Skip to content

S
slapos.buildout
Commit 1aa316f4 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Merge branch 'bug_778748'

parents d7fa2df1 b129d3e7
Show whitespace changes
Inline Side-by-side
Showing with 182 additions and 9 deletions
src/zc/buildout/buildout.py
View file @ 1aa316f4
......@@ -355,6 +355,10 @@ class Buildout(UserDict.DictMixin):
self.upload_dir_url = None
self.signature_private_key_file = None
self.signature_certificate_list = None
self.shacache_cert_file = None
self.shacache_key_file = None
self.shadir_cert_file = None
self.shadir_key_file = None
else:
# support networkcache
networkcache_section_name = options.get('networkcache-section')
......@@ -372,6 +376,14 @@ class Buildout(UserDict.DictMixin):
'signature-certificate-list', '')
self.signature_private_key_file = networkcache_section.get(
'signature-private-key-file', '')
self.shacache_cert_file = networkcache_section.get(
'shacache-cert-file', '')
self.shacache_key_file = networkcache_section.get(
'shacache-key-file', '')
self.shadir_cert_file = networkcache_section.get(
'shadir-cert-file', '')
self.shadir_key_file = networkcache_section.get(
'shadir-key-file', '')
# parse signature list
cert_marker = '-----BEGIN CERTIFICATE-----'
......@@ -391,6 +403,10 @@ class Buildout(UserDict.DictMixin):
options['__networkcache__upload-dir-url'] = self.upload_dir_url
options['__networkcache__signature-certificate-list'] = signature_certificate_list
options['__networkcache__signature-private-key-file'] = self.signature_private_key_file
options['__networkcache__shacache-cert-file'] = self.shacache_cert_file
options['__networkcache__shacache-key-file'] = self.shacache_key_file
options['__networkcache__shadir-cert-file'] = self.shadir_cert_file
options['__networkcache__shadir-key-file'] = self.shadir_key_file
else:
self._logger.debug('Networkcache functionality not enabled. '
......@@ -401,6 +417,10 @@ class Buildout(UserDict.DictMixin):
self.upload_dir_url = None
self.signature_private_key_file = None
self.signature_certificate_list = None
self.shacache_cert_file = None
self.shacache_key_file = None
self.shadir_cert_file = None
self.shadir_key_file = None
def _buildout_path(self, name):
if '${' in name:
......@@ -442,6 +462,10 @@ class Buildout(UserDict.DictMixin):
upload_dir_url=self.upload_dir_url,
signature_private_key_file=self.signature_private_key_file,
signature_certificate_list=self.signature_certificate_list,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,
)
# Now copy buildout and setuptools eggs, and record destination eggs:
......@@ -954,6 +978,10 @@ class Buildout(UserDict.DictMixin):
upload_dir_url=self.upload_dir_url,
signature_certificate_list=self.signature_certificate_list,
signature_private_key_file=self.signature_private_key_file,
shacache_cert_file=self.shacache_cert_file,
shacache_key_file=self.shacache_key_file,
shadir_cert_file=self.shadir_cert_file,
shadir_key_file=self.shadir_key_file,
)
upgraded = []
......@@ -1193,6 +1221,14 @@ def _install_and_load(spec, group, entry, buildout):
'__networkcache__signature-private-key-file'),
signature_certificate_list=buildout_options.get(
'__networkcache__signature-certificate-list'),
shacache_cert_file=buildout_options.get(
'__networkcache__shacache-cert-file'),
shacache_key_file=buildout_options.get(
'__networkcache__shacache-key-file'),
shadir_cert_file=buildout_options.get(
'__networkcache__shadir-cert-file'),
shadir_key_file=buildout_options.get(
'__networkcache__shadir-key-file'),
)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
......
src/zc/buildout/download.py
View file @ 1aa316f4
......@@ -86,6 +86,14 @@ class Download(object):
if q.strip()]
self.signature_private_key_file = options.get(
'__networkcache__signature-private-key-file')
self.shacache_cert_file = options.get(
'__networkcache__shacache-cert-file')
self.shacache_key_file = options.get(
'__networkcache__shacache-key-file')
self.shadir_cert_file = options.get(
'__networkcache__shadir-cert-file')
self.shadir_key_file = options.get(
'__networkcache__shadir-key-file')
@property
def download_cache(self):
......@@ -206,7 +214,11 @@ class Download(object):
if self.upload_cache_url and self.upload_dir_url:
upload_network_cached(self.upload_dir_url,
self.upload_cache_url, url, tmp_path, self.logger,
self.signature_private_key_file)
self.signature_private_key_file,
self.shacache_cert_file,
self.shacache_key_file,
self.shadir_cert_file,
self.shadir_key_file)
finally:
os.close(handle)
except:
......
src/zc/buildout/easy_install.py
View file @ 1aa316f4
......@@ -346,7 +346,11 @@ class Installer:
upload_dir_url=None,
upload_cache_url=None,
signature_certificate_list=None,
signature_private_key_file=None
signature_private_key_file=None,
shacache_cert_file=None,
shacache_key_file=None,
shadir_cert_file=None,
shadir_key_file=None,
):
self._dest = dest
self._allow_hosts = allow_hosts
......@@ -421,6 +425,10 @@ class Installer:
self._upload_cache_url = upload_cache_url
self._signature_certificate_list = signature_certificate_list
self._signature_private_key_file = signature_private_key_file
self._shacache_cert_file = shacache_cert_file
self._shacache_key_file = shacache_key_file
self._shadir_cert_file = shadir_cert_file
self._shadir_key_file = shadir_key_file
_allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name):
......@@ -730,7 +738,9 @@ class Installer:
if self._upload_cache_url and self._upload_dir_url:
upload_network_cached(self._upload_dir_url,
self._upload_cache_url, dist.location, new_location, logger,
self._signature_private_key_file)
self._signature_private_key_file,
self._shacache_cert_file, self._shacache_key_file,
self._shadir_cert_file, self._shadir_key_file)
if (download_cache
and (realpath(new_location) == realpath(dist.location))
......@@ -1111,7 +1121,9 @@ def install(specs, dest,
include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, download_dir_url=None, download_cache_url=None,
upload_dir_url=None, upload_cache_url=None,
signature_certificate_list=None, signature_private_key_file=None):
signature_certificate_list=None, signature_private_key_file=None,
shacache_cert_file=None, shacache_key_file=None,
shadir_cert_file=None, shadir_key_file=None):
installer = Installer(
dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts,
......@@ -1121,7 +1133,11 @@ def install(specs, dest,
download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
upload_cache_url=upload_cache_url,
signature_certificate_list=signature_certificate_list,
signature_private_key_file=signature_private_key_file)
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
return installer.install(specs, working_set)
......
src/zc/buildout/networkcache.py
View file @ 1aa316f4
......@@ -105,7 +105,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_private_key_file):
signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file):
"""Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED:
return False
......@@ -122,12 +123,24 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
urlmd5=hashlib.md5(external_url).hexdigest())
f = open(path, 'r')
if not signature_private_key_file:
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
signature_private_key_file = None
if not shacache_cert_file:
shacache_cert_file = None
if not shacache_key_file:
shacache_key_file = None
if not shadir_cert_file:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
try:
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file)
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
except TypeError:
logger.warning('Incompatible version of networkcache, not using it.')
return False
......
src/zc/buildout/networkcache.txt
View file @ 1aa316f4
......@@ -693,6 +693,97 @@ Of course eggs are also supported by networkcache:
Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
Got demoneeded 1.2c1.
It is possible to pass SSL certificates and key to be used by authentication.
In order to do it NetworkcacheClient.upload method have to be patched to show
that proper keys are used.
>>> mkdir(sample_buildout, 'ncpatch')
>>> write(sample_buildout, 'ncpatch', 'ncpatch.py',
... """
... def patched(self, *args, **kwargs):
... print 'shacache_cert_file', self.shacache_cert_file
... print 'shacache_key_file', self.shacache_key_file
... print 'shadir_cert_file', self.shadir_cert_file
... print 'shadir_key_file', self.shadir_key_file
... return False
...
... def ext(buildout):
... print 'Patching slapos.libnetworkcache'
... import slapos.libnetworkcache
... slapos.libnetworkcache.NetworkcacheClient.upload = patched
... """)
>>> write(sample_buildout, 'ncpatch', 'setup.py',
... """
... from setuptools import setup
...
... setup(
... name = "ncpatch",
... entry_points = {'zc.buildout.extension': ['ncpatch = ncpatch:ext']},
... )
... """)
>>> write(sample_buildout, 'ncpatch', 'README.txt', " ")
Lets activiate this extension:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = ncpatch
... parts =
... """)
>>> print system(buildout)
Develop: '/sample-buildout/ncpatch'
Uninstalling eggs.
Lets clean networkcache:
>>> rmdir(nc_data)
>>> mkdir(nc_data)
And see what happens:
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
...
... networkcache-section = networkcache
... find-links = %(link_server)s
... develop = ncpatch download
... extensions = ncpatch
... parts = download
...
... [download]
... recipe = download
... url = %(remote_server_url)shello.txt
...
... [networkcache]
... download-cache-url = %(nc_url)sshacache
... download-dir-url = %(nc_url)sshadir
... upload-cache-url = %(nc_url)sshacache
... upload-dir-url = %(nc_url)sshadir
... shacache-cert-file = /path/to/shacache/cert/file
... shacache-key-file = /path/to/shacache/key/file
... shadir-cert-file = /path/to/shadir/cert/file
... shadir-key-file = /path/to/shadir/key/file
... ''' % globals())
>>> print system(buildout)
Networkcache enabled.
Networkcache download cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Networkcache upload cache: 'http://localhost/shacache', directory 'http://localhost/shadir'
Patching slapos.libnetworkcache
Develop: '/sample-buildout/ncpatch'
Develop: '/sample-buildout/download'
Installing download.
Downloading http://localhost/hello.txt
Downloading hello.txt from network cache.
Failed to download from network cache hello.txt:...404...
Uploading http://localhost/hello.txt into network cache.
shacache_cert_file /path/to/shacache/cert/file
shacache_key_file /path/to/shacache/key/file
shadir_cert_file /path/to/shadir/cert/file
shadir_key_file /path/to/shadir/key/file
download: Downloaded http://localhost/hello.txt
Lets clean networkcache:
>>> rmdir(nc_data)
......
zc.recipe.egg_/src/zc/recipe/egg/egg.py
View file @ 1aa316f4
......@@ -94,7 +94,12 @@ class Eggs(object):
('__networkcache__upload-cache-url', 'upload_cache_url'),
('__networkcache__upload-dir-url', 'upload_dir_url'),
('__networkcache__signature-private-key-file', 'signature_private_key_file'),
('__networkcache__signature-certificate-list', 'signature_certificate_list')):
('__networkcache__signature-certificate-list', 'signature_certificate_list'),
('__networkcache__shacache-cert-file', 'shacache_cert_file'),
('__networkcache__shacache-key-file', 'shacache_key_file'),
('__networkcache__shadir-cert-file', 'shadir_cert_file'),
('__networkcache__shadir-key-file', 'shadir_key_file'),
):
if option_key in b_options:
if option_key == '__networkcache__signature-certificate-list':
cert_marker = '-----BEGIN CERTIFICATE-----'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7