Commit 297bbd6f authored by Tarek Ziad's avatar Tarek Ziad

merged the allow-hosts option

parent 96569be0
......@@ -7,6 +7,8 @@ Change History
1.0.4 (unreleased)
==================
- Added the `allow-hosts` option (tarek)
- Quote the 'executable' argument when trying to detect the python
version using popen4. (sidnei)
......
Allow hosts
-----------
On some environments the links visited by `zc.buildout` can be forbidden
by paranoiac firewalls. These URL might be on the chain of links
visited by `zc.buildout` wheter they are defined in the `find-links` option,
wheter they are defined by various eggs in their `url`, `download_url`,
`dependency_links` metadata.
It is even harder to track that package_index works like a spider and
might visit links and go to other location.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`
(see `easy_install allow-hosts option`_).
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
Let's create a develop egg in our buildout that specifies
`dependency_links` which points to a server in the outside world::
>>> mkdir(sample_buildout, 'allowdemo')
>>> write(sample_buildout, 'allowdemo', 'dependencydemo.py',
... 'import eggrecipekss.core')
>>> write(sample_buildout, 'allowdemo', 'setup.py',
... '''from setuptools import setup; setup(
... name='allowdemo', py_modules=['dependencydemo'],
... install_requires = 'kss.core',
... dependency_links = ['http://dist.plone.org'],
... zip_safe=True, version='1')
... ''')
Now let's configure the buildout to use the develop egg,
together with some rules that disallow any website but PyPI and
local files::
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = allowdemo
... parts = eggs
... allow-hosts =
... pypi.python.org
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = allowdemo
... ''')
Now we can run the buildout and make sure all attempts to dist.plone.org fails::
>>> print system(buildout)
Develop: '/sample-buildout/allowdemo'
Installing eggs.
<BLANKLINE>
Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
<BLANKLINE>
Couldn't find index page for 'kss.core' (maybe misspelled?)
Getting distribution for 'kss.core'.
While:
Installing eggs.
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
<BLANKLINE>
That's what we wanted : this will prevent any attempt to access
unwanted domains. For instance, some packages are listing in their
links `svn://` links. These can lead to error in some cases, and
can therefore be protected like this::
XXX (showcase with a svn:// file)
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = allowdemo
... parts = eggs
... allow-hosts =
... ^(!svn://).*
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = allowdemo
... ''')
Now we can run the buildout and make sure all attempts to dist.plone.org fails::
>>> print system(buildout)
Develop: '/sample-buildout/allowdemo'
Installing eggs.
<BLANKLINE>
Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
<BLANKLINE>
Couldn't find index page for 'kss.core' (maybe misspelled?)
Getting distribution for 'kss.core'.
While:
Installing eggs.
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
<BLANKLINE>
......@@ -136,6 +136,10 @@ class Buildout(UserDict.DictMixin):
links = options.get('find-links', '')
self._links = links and links.split() or ()
allow_hosts = options.get('allow-hosts', '*').split('\n')
self._allow_hosts = tuple([host.strip() for host in allow_hosts
if host.strip() != ''])
self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
......@@ -665,7 +669,8 @@ class Buildout(UserDict.DictMixin):
self['buildout']['eggs-directory'],
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
path = [self['buildout']['develop-eggs-directory']]
path = [self['buildout']['develop-eggs-directory']],
allow_hosts = self._allow_hosts
)
upgraded = []
......@@ -743,7 +748,7 @@ class Buildout(UserDict.DictMixin):
working_set=pkg_resources.working_set,
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
newest=self.newest)
newest=self.newest, allow_hosts=self._allow_hosts)
# Clear cache because extensions might now let us read pages we
# couldn't read before.
......@@ -836,6 +841,7 @@ def _install_and_load(spec, group, entry, buildout):
path=path,
working_set=pkg_resources.working_set,
newest=buildout.newest,
allow_hosts=buildout._allow_hosts
)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
......
......@@ -2385,7 +2385,31 @@ We see that our extension is loaded and executed:
ext ['buildout']
Develop: '/sample-bootstrapped/demo'
Allow hosts
-----------
On some environments the links visited by `zc.buildout` can be forbidden
by paranoiac firewalls. These URL might be on the chain of links
visited by `zc.buildout` wheter they are defined in the `find-links` option,
wheter they are defined by various eggs in their `url`, `download_url`,
`dependency_links` metadata.
It is even harder to track that package_index works like a spider and
might visit links and go to other location.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`.
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
All urls that does not match these hosts will not be visited.
.. [#future_recipe_methods] In the future, additional methods may be
added. Older recipes with fewer methods will still be
......
......@@ -71,8 +71,21 @@ def _get_version(executable):
_versions[executable] = version
return version
FILE_SCHEME = re.compile('file://', re.I).match
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
"""Will allow urls that are local to the system.
No matter what is allow_hosts.
"""
def url_ok(self, url, fatal=False):
if FILE_SCHEME(url):
return True
return setuptools.package_index.PackageIndex.url_ok(self, url, False)
_indexes = {}
def _get_index(executable, index_url, find_links):
def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
key = executable, index_url, tuple(find_links)
index = _indexes.get(key)
if index is not None:
......@@ -80,9 +93,8 @@ def _get_index(executable, index_url, find_links):
if index_url is None:
index_url = default_index_url
index = setuptools.package_index.PackageIndex(
index_url, python=_get_version(executable)
index = AllowHostsPackageIndex(
index_url, hosts=allow_hosts, python=_get_version(executable)
)
if find_links:
......@@ -124,8 +136,10 @@ class Installer:
newest=True,
versions=None,
use_dependency_links=None,
allow_hosts=('*',)
):
self._dest = dest
self._allow_hosts = allow_hosts
if self._install_from_cache:
if not self._download_cache:
......@@ -152,7 +166,7 @@ class Installer:
self._newest = newest
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
self._index = _get_index(executable, index, links)
self._index = _get_index(executable, index, links, self._allow_hosts)
if versions is not None:
self._versions = versions
......@@ -162,9 +176,9 @@ class Installer:
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
return None, self._obtain(req, source)
# Note that dists are sorted from best to worst, as promised by
# env.__getitem__
......@@ -345,9 +359,9 @@ class Installer:
shutil.rmtree(tmp)
def _obtain(self, requirement, source=None):
# initialize out index for this project:
index = self._index
if index.obtain(requirement) is None:
# Nothing is available.
return None
......@@ -425,7 +439,7 @@ class Installer:
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist, avail = self._satisfied(requirement)
if dist is None:
if self._dest is not None:
logger.info(*__doing__)
......@@ -514,7 +528,8 @@ class Installer:
logger.debug('Adding find link %r from %s', link, dist)
self._links.append(link)
self._index = _get_index(self._executable,
self._index_url, self._links)
self._index_url, self._links,
self._allow_hosts)
for dist in dists:
# Check whether we picked a version and, if we did, report it:
......@@ -729,18 +744,19 @@ def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=False,
path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None):
use_dependency_links=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links)
newest, versions, use_dependency_links,
allow_hosts=allow_hosts)
return installer.install(specs, working_set)
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
path=None, newest=True, versions=None):
path=None, newest=True, versions=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
versions)
versions, allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
......
......@@ -2657,6 +2657,7 @@ def test_suite():
doctest.DocFileSuite(
'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt',
'allowhosts.txt',
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
......
......@@ -39,6 +39,11 @@ class Eggs(object):
options['index'] = index
self.index = index
allow_hosts = buildout['buildout'].get('allow-hosts', '*')
allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
if host.strip()!=''])
self.allow_hosts = allow_hosts
options['eggs-directory'] = buildout['buildout']['eggs-directory']
options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory'
......@@ -78,6 +83,7 @@ class Eggs(object):
always_unzip=options.get('unzip') == 'true',
path=[options['develop-eggs-directory']],
newest=self.buildout['buildout'].get('newest') == 'true',
allow_hosts=self.allow_hosts
)
return orig_distributions, ws
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment