Commit 297bbd6f authored by Tarek Ziad's avatar Tarek Ziad

merged the allow-hosts option

parent 96569be0
...@@ -7,6 +7,8 @@ Change History ...@@ -7,6 +7,8 @@ Change History
1.0.4 (unreleased) 1.0.4 (unreleased)
================== ==================
- Added the `allow-hosts` option (tarek)
- Quote the 'executable' argument when trying to detect the python - Quote the 'executable' argument when trying to detect the python
version using popen4. (sidnei) version using popen4. (sidnei)
......
Allow hosts
-----------
On some environments the links visited by `zc.buildout` can be forbidden
by paranoiac firewalls. These URL might be on the chain of links
visited by `zc.buildout` wheter they are defined in the `find-links` option,
wheter they are defined by various eggs in their `url`, `download_url`,
`dependency_links` metadata.
It is even harder to track that package_index works like a spider and
might visit links and go to other location.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`
(see `easy_install allow-hosts option`_).
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
Let's create a develop egg in our buildout that specifies
`dependency_links` which points to a server in the outside world::
>>> mkdir(sample_buildout, 'allowdemo')
>>> write(sample_buildout, 'allowdemo', 'dependencydemo.py',
... 'import eggrecipekss.core')
>>> write(sample_buildout, 'allowdemo', 'setup.py',
... '''from setuptools import setup; setup(
... name='allowdemo', py_modules=['dependencydemo'],
... install_requires = 'kss.core',
... dependency_links = ['http://dist.plone.org'],
... zip_safe=True, version='1')
... ''')
Now let's configure the buildout to use the develop egg,
together with some rules that disallow any website but PyPI and
local files::
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = allowdemo
... parts = eggs
... allow-hosts =
... pypi.python.org
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = allowdemo
... ''')
Now we can run the buildout and make sure all attempts to dist.plone.org fails::
>>> print system(buildout)
Develop: '/sample-buildout/allowdemo'
Installing eggs.
<BLANKLINE>
Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
<BLANKLINE>
Couldn't find index page for 'kss.core' (maybe misspelled?)
Getting distribution for 'kss.core'.
While:
Installing eggs.
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
<BLANKLINE>
That's what we wanted : this will prevent any attempt to access
unwanted domains. For instance, some packages are listing in their
links `svn://` links. These can lead to error in some cases, and
can therefore be protected like this::
XXX (showcase with a svn:// file)
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = allowdemo
... parts = eggs
... allow-hosts =
... ^(!svn://).*
...
... [eggs]
... recipe = zc.recipe.egg:eggs
... eggs = allowdemo
... ''')
Now we can run the buildout and make sure all attempts to dist.plone.org fails::
>>> print system(buildout)
Develop: '/sample-buildout/allowdemo'
Installing eggs.
<BLANKLINE>
Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
<BLANKLINE>
Couldn't find index page for 'kss.core' (maybe misspelled?)
Getting distribution for 'kss.core'.
While:
Installing eggs.
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
<BLANKLINE>
...@@ -136,6 +136,10 @@ class Buildout(UserDict.DictMixin): ...@@ -136,6 +136,10 @@ class Buildout(UserDict.DictMixin):
links = options.get('find-links', '') links = options.get('find-links', '')
self._links = links and links.split() or () self._links = links and links.split() or ()
allow_hosts = options.get('allow-hosts', '*').split('\n')
self._allow_hosts = tuple([host.strip() for host in allow_hosts
if host.strip() != ''])
self._buildout_dir = options['directory'] self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs', 'develop-eggs'): for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
...@@ -665,7 +669,8 @@ class Buildout(UserDict.DictMixin): ...@@ -665,7 +669,8 @@ class Buildout(UserDict.DictMixin):
self['buildout']['eggs-directory'], self['buildout']['eggs-directory'],
links = self['buildout'].get('find-links', '').split(), links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'), index = self['buildout'].get('index'),
path = [self['buildout']['develop-eggs-directory']] path = [self['buildout']['develop-eggs-directory']],
allow_hosts = self._allow_hosts
) )
upgraded = [] upgraded = []
...@@ -743,7 +748,7 @@ class Buildout(UserDict.DictMixin): ...@@ -743,7 +748,7 @@ class Buildout(UserDict.DictMixin):
working_set=pkg_resources.working_set, working_set=pkg_resources.working_set,
links = self['buildout'].get('find-links', '').split(), links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'), index = self['buildout'].get('index'),
newest=self.newest) newest=self.newest, allow_hosts=self._allow_hosts)
# Clear cache because extensions might now let us read pages we # Clear cache because extensions might now let us read pages we
# couldn't read before. # couldn't read before.
...@@ -836,6 +841,7 @@ def _install_and_load(spec, group, entry, buildout): ...@@ -836,6 +841,7 @@ def _install_and_load(spec, group, entry, buildout):
path=path, path=path,
working_set=pkg_resources.working_set, working_set=pkg_resources.working_set,
newest=buildout.newest, newest=buildout.newest,
allow_hosts=buildout._allow_hosts
) )
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
......
...@@ -2385,7 +2385,31 @@ We see that our extension is loaded and executed: ...@@ -2385,7 +2385,31 @@ We see that our extension is loaded and executed:
ext ['buildout'] ext ['buildout']
Develop: '/sample-bootstrapped/demo' Develop: '/sample-bootstrapped/demo'
Allow hosts
-----------
On some environments the links visited by `zc.buildout` can be forbidden
by paranoiac firewalls. These URL might be on the chain of links
visited by `zc.buildout` wheter they are defined in the `find-links` option,
wheter they are defined by various eggs in their `url`, `download_url`,
`dependency_links` metadata.
It is even harder to track that package_index works like a spider and
might visit links and go to other location.
The `allow-hosts` option provides a way to prevent this, and
works exactly like the one provided in `easy_install`.
You can provide a list of allowed host, together with wildcards::
[buildout]
...
allow-hosts =
*.python.org
example.com
All urls that does not match these hosts will not be visited.
.. [#future_recipe_methods] In the future, additional methods may be .. [#future_recipe_methods] In the future, additional methods may be
added. Older recipes with fewer methods will still be added. Older recipes with fewer methods will still be
......
...@@ -71,8 +71,21 @@ def _get_version(executable): ...@@ -71,8 +71,21 @@ def _get_version(executable):
_versions[executable] = version _versions[executable] = version
return version return version
FILE_SCHEME = re.compile('file://', re.I).match
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
"""Will allow urls that are local to the system.
No matter what is allow_hosts.
"""
def url_ok(self, url, fatal=False):
if FILE_SCHEME(url):
return True
return setuptools.package_index.PackageIndex.url_ok(self, url, False)
_indexes = {} _indexes = {}
def _get_index(executable, index_url, find_links): def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
key = executable, index_url, tuple(find_links) key = executable, index_url, tuple(find_links)
index = _indexes.get(key) index = _indexes.get(key)
if index is not None: if index is not None:
...@@ -80,9 +93,8 @@ def _get_index(executable, index_url, find_links): ...@@ -80,9 +93,8 @@ def _get_index(executable, index_url, find_links):
if index_url is None: if index_url is None:
index_url = default_index_url index_url = default_index_url
index = AllowHostsPackageIndex(
index = setuptools.package_index.PackageIndex( index_url, hosts=allow_hosts, python=_get_version(executable)
index_url, python=_get_version(executable)
) )
if find_links: if find_links:
...@@ -124,8 +136,10 @@ class Installer: ...@@ -124,8 +136,10 @@ class Installer:
newest=True, newest=True,
versions=None, versions=None,
use_dependency_links=None, use_dependency_links=None,
allow_hosts=('*',)
): ):
self._dest = dest self._dest = dest
self._allow_hosts = allow_hosts
if self._install_from_cache: if self._install_from_cache:
if not self._download_cache: if not self._download_cache:
...@@ -152,7 +166,7 @@ class Installer: ...@@ -152,7 +166,7 @@ class Installer:
self._newest = newest self._newest = newest
self._env = pkg_resources.Environment(path, self._env = pkg_resources.Environment(path,
python=_get_version(executable)) python=_get_version(executable))
self._index = _get_index(executable, index, links) self._index = _get_index(executable, index, links, self._allow_hosts)
if versions is not None: if versions is not None:
self._versions = versions self._versions = versions
...@@ -162,9 +176,9 @@ class Installer: ...@@ -162,9 +176,9 @@ class Installer:
if not dists: if not dists:
logger.debug('We have no distributions for %s that satisfies %r.', logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req)) req.project_name, str(req))
return None, self._obtain(req, source) return None, self._obtain(req, source)
# Note that dists are sorted from best to worst, as promised by # Note that dists are sorted from best to worst, as promised by
# env.__getitem__ # env.__getitem__
...@@ -345,9 +359,9 @@ class Installer: ...@@ -345,9 +359,9 @@ class Installer:
shutil.rmtree(tmp) shutil.rmtree(tmp)
def _obtain(self, requirement, source=None): def _obtain(self, requirement, source=None):
# initialize out index for this project: # initialize out index for this project:
index = self._index index = self._index
if index.obtain(requirement) is None: if index.obtain(requirement) is None:
# Nothing is available. # Nothing is available.
return None return None
...@@ -425,7 +439,7 @@ class Installer: ...@@ -425,7 +439,7 @@ class Installer:
# Maybe an existing dist is already the best dist that satisfies the # Maybe an existing dist is already the best dist that satisfies the
# requirement # requirement
dist, avail = self._satisfied(requirement) dist, avail = self._satisfied(requirement)
if dist is None: if dist is None:
if self._dest is not None: if self._dest is not None:
logger.info(*__doing__) logger.info(*__doing__)
...@@ -514,7 +528,8 @@ class Installer: ...@@ -514,7 +528,8 @@ class Installer:
logger.debug('Adding find link %r from %s', link, dist) logger.debug('Adding find link %r from %s', link, dist)
self._links.append(link) self._links.append(link)
self._index = _get_index(self._executable, self._index = _get_index(self._executable,
self._index_url, self._links) self._index_url, self._links,
self._allow_hosts)
for dist in dists: for dist in dists:
# Check whether we picked a version and, if we did, report it: # Check whether we picked a version and, if we did, report it:
...@@ -729,18 +744,19 @@ def install(specs, dest, ...@@ -729,18 +744,19 @@ def install(specs, dest,
links=(), index=None, links=(), index=None,
executable=sys.executable, always_unzip=False, executable=sys.executable, always_unzip=False,
path=None, working_set=None, newest=True, versions=None, path=None, working_set=None, newest=True, versions=None,
use_dependency_links=None): use_dependency_links=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path, installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links) newest, versions, use_dependency_links,
allow_hosts=allow_hosts)
return installer.install(specs, working_set) return installer.install(specs, working_set)
def build(spec, dest, build_ext, def build(spec, dest, build_ext,
links=(), index=None, links=(), index=None,
executable=sys.executable, executable=sys.executable,
path=None, newest=True, versions=None): path=None, newest=True, versions=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest, installer = Installer(dest, links, index, executable, True, path, newest,
versions) versions, allow_hosts=allow_hosts)
return installer.build(spec, build_ext) return installer.build(spec, build_ext)
......
...@@ -2657,6 +2657,7 @@ def test_suite(): ...@@ -2657,6 +2657,7 @@ def test_suite():
doctest.DocFileSuite( doctest.DocFileSuite(
'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt', 'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt',
'allowhosts.txt',
setUp=easy_install_SetUp, setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown, tearDown=zc.buildout.testing.buildoutTearDown,
......
...@@ -39,6 +39,11 @@ class Eggs(object): ...@@ -39,6 +39,11 @@ class Eggs(object):
options['index'] = index options['index'] = index
self.index = index self.index = index
allow_hosts = buildout['buildout'].get('allow-hosts', '*')
allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
if host.strip()!=''])
self.allow_hosts = allow_hosts
options['eggs-directory'] = buildout['buildout']['eggs-directory'] options['eggs-directory'] = buildout['buildout']['eggs-directory']
options['_e'] = options['eggs-directory'] # backward compat. options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory' options['develop-eggs-directory'
...@@ -78,6 +83,7 @@ class Eggs(object): ...@@ -78,6 +83,7 @@ class Eggs(object):
always_unzip=options.get('unzip') == 'true', always_unzip=options.get('unzip') == 'true',
path=[options['develop-eggs-directory']], path=[options['develop-eggs-directory']],
newest=self.buildout['buildout'].get('newest') == 'true', newest=self.buildout['buildout'].get('newest') == 'true',
allow_hosts=self.allow_hosts
) )
return orig_distributions, ws return orig_distributions, ws
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment