Commit c69a72f9 authored by Lucas Carvalho's avatar Lucas Carvalho Committed by Julien Muchembled

Add new params: signature_public_file & signature_private_file

Both files must be generated by opensssl.
- signature_public_file: used to verify if the content is trustable by
  libnetworkcache
- signature_private_file: used to generate the signature before uploading
  the content to shadir by libnetworkcache.
parent 41062088
...@@ -343,6 +343,8 @@ class Buildout(UserDict.DictMixin): ...@@ -343,6 +343,8 @@ class Buildout(UserDict.DictMixin):
self.download_dir_url = None self.download_dir_url = None
self.upload_cache_url = None self.upload_cache_url = None
self.upload_dir_url = None self.upload_dir_url = None
self.signature_private_file = None
self.signature_public_file = None
else: else:
# support networkcache # support networkcache
networkcache_section_name = options.get('networkcache-section') networkcache_section_name = options.get('networkcache-section')
...@@ -356,6 +358,11 @@ class Buildout(UserDict.DictMixin): ...@@ -356,6 +358,11 @@ class Buildout(UserDict.DictMixin):
'upload-cache-url', '') 'upload-cache-url', '')
self.upload_dir_url = networkcache_section.get( self.upload_dir_url = networkcache_section.get(
'upload-dir-url', '') 'upload-dir-url', '')
self.signature_private_file = options.get(
'signature_private_file', '')
self.signature_public_file = options.get(
'signature_public_file', '')
self._logger.info('Networkcache enabled.') self._logger.info('Networkcache enabled.')
self._logger.info('Networkcache download cache: %r, directory ' self._logger.info('Networkcache download cache: %r, directory '
'%r' % (self.download_cache_url, self.download_dir_url)) '%r' % (self.download_cache_url, self.download_dir_url))
...@@ -373,7 +380,8 @@ class Buildout(UserDict.DictMixin): ...@@ -373,7 +380,8 @@ class Buildout(UserDict.DictMixin):
self.download_dir_url = None self.download_dir_url = None
self.upload_cache_url = None self.upload_cache_url = None
self.upload_dir_url = None self.upload_dir_url = None
self.signature_private_file = None
self.signature_public_file = None
def _buildout_path(self, name): def _buildout_path(self, name):
if '${' in name: if '${' in name:
...@@ -412,7 +420,9 @@ class Buildout(UserDict.DictMixin): ...@@ -412,7 +420,9 @@ class Buildout(UserDict.DictMixin):
download_cache_url=self.download_cache_url, download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url, download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url, upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url upload_dir_url=self.upload_dir_url,
signature_private_file=self.signature_private_file,
signature_public_file=self.signature_public_file,
) )
# Now copy buildout and setuptools eggs, and record destination eggs: # Now copy buildout and setuptools eggs, and record destination eggs:
...@@ -917,7 +927,9 @@ class Buildout(UserDict.DictMixin): ...@@ -917,7 +927,9 @@ class Buildout(UserDict.DictMixin):
download_cache_url=self.download_cache_url, download_cache_url=self.download_cache_url,
download_dir_url=self.download_dir_url, download_dir_url=self.download_dir_url,
upload_cache_url=self.upload_cache_url, upload_cache_url=self.upload_cache_url,
upload_dir_url=self.upload_dir_url upload_dir_url=self.upload_dir_url,
signature_public_file=self.signature_public_file,
signature_private_file=self.signature_private_file,
) )
upgraded = [] upgraded = []
...@@ -1152,7 +1164,9 @@ def _install_and_load(spec, group, entry, buildout): ...@@ -1152,7 +1164,9 @@ def _install_and_load(spec, group, entry, buildout):
'__networkcache__download-dir-url'), '__networkcache__download-dir-url'),
upload_cache_url=buildout_options.get( upload_cache_url=buildout_options.get(
'__networkcache__upload-cache-url'), '__networkcache__upload-cache-url'),
upload_dir_url=buildout_options.get('__networkcache__upload-dir-url') upload_dir_url=buildout_options.get('__networkcache__upload-dir-url'),
signature_private_file=buildout_options.get('signature_private_file'),
signature_public_file=buildout_options.get('signature_public_file'),
) )
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
......
...@@ -77,6 +77,10 @@ class Download(object): ...@@ -77,6 +77,10 @@ class Download(object):
self.upload_dir_url = options.get('__networkcache__upload-dir-url') self.upload_dir_url = options.get('__networkcache__upload-dir-url')
self.upload_cache_url = options.get( self.upload_cache_url = options.get(
'__networkcache__upload-cache-url') '__networkcache__upload-cache-url')
self.signature_public_file = options.get(
'signature_public_file')
self.signature_private_file = options.get(
'signature_private_file')
@property @property
def download_cache(self): def download_cache(self):
...@@ -182,9 +186,10 @@ class Download(object): ...@@ -182,9 +186,10 @@ class Download(object):
try: try:
try: try:
if not download_network_cached(self.download_dir_url, self.download_cache_url, if not download_network_cached(self.download_dir_url,
tmp_path, url, self.logger, md5sum): self.download_cache_url, tmp_path, url, self.logger,
self.signature_public_file,
self.signature_private_file, md5sum):
# Download from original url # Download from original url
tmp_path, headers = urllib.urlretrieve(url, tmp_path) tmp_path, headers = urllib.urlretrieve(url, tmp_path)
if not check_md5sum(tmp_path, md5sum): if not check_md5sum(tmp_path, md5sum):
...@@ -193,7 +198,9 @@ class Download(object): ...@@ -193,7 +198,9 @@ class Download(object):
# Upload the file to networkcached. # Upload the file to networkcached.
if self.upload_cache_url and self.upload_dir_url: if self.upload_cache_url and self.upload_dir_url:
upload_network_cached(self.upload_dir_url, upload_network_cached(self.upload_dir_url,
self.upload_cache_url, url, tmp_path, self.logger) self.upload_cache_url, url, tmp_path, self.logger,
self.signature_public_file,
self.signature_private_file)
finally: finally:
os.close(handle) os.close(handle)
except: except:
......
...@@ -344,7 +344,9 @@ class Installer: ...@@ -344,7 +344,9 @@ class Installer:
download_dir_url=None, download_dir_url=None,
download_cache_url=None, download_cache_url=None,
upload_dir_url=None, upload_dir_url=None,
upload_cache_url=None upload_cache_url=None,
signature_public_file=None,
signature_private_file=None
): ):
self._dest = dest self._dest = dest
self._allow_hosts = allow_hosts self._allow_hosts = allow_hosts
...@@ -417,6 +419,8 @@ class Installer: ...@@ -417,6 +419,8 @@ class Installer:
self._download_cache_url = download_cache_url self._download_cache_url = download_cache_url
self._upload_dir_url = upload_dir_url self._upload_dir_url = upload_dir_url
self._upload_cache_url = upload_cache_url self._upload_cache_url = upload_cache_url
self._signature_public_file = signature_public_file
self._signature_private_file = signature_private_file
_allowed_eggs_from_site_packages_regex = None _allowed_eggs_from_site_packages_regex = None
def allow_site_package_egg(self, name): def allow_site_package_egg(self, name):
...@@ -719,12 +723,14 @@ class Installer: ...@@ -719,12 +723,14 @@ class Installer:
filename = get_filename_from_url(dist.location) filename = get_filename_from_url(dist.location)
new_location = os.path.join(tmp, filename) new_location = os.path.join(tmp, filename)
if not download_network_cached(self._download_dir_url, self._download_cache_url, if not download_network_cached(self._download_dir_url,
new_location, dist.location, logger): self._download_cache_url, new_location, dist.location, logger,
self._signature_public_file, self._signature_private_file):
new_location = self._index.download(dist.location, tmp) new_location = self._index.download(dist.location, tmp)
if self._upload_cache_url and self._upload_dir_url: if self._upload_cache_url and self._upload_dir_url:
upload_network_cached(self._upload_dir_url, upload_network_cached(self._upload_dir_url,
self._upload_cache_url, dist.location, new_location, logger) self._upload_cache_url, dist.location, new_location, logger,
self._signature_public_file, self._signature_private_file)
if (download_cache if (download_cache
and (realpath(new_location) == realpath(dist.location)) and (realpath(new_location) == realpath(dist.location))
...@@ -1104,7 +1110,8 @@ def install(specs, dest, ...@@ -1104,7 +1110,8 @@ def install(specs, dest,
use_dependency_links=None, allow_hosts=('*',), use_dependency_links=None, allow_hosts=('*',),
include_site_packages=None, allowed_eggs_from_site_packages=None, include_site_packages=None, allowed_eggs_from_site_packages=None,
prefer_final=None, download_dir_url=None, download_cache_url=None, prefer_final=None, download_dir_url=None, download_cache_url=None,
upload_dir_url=None, upload_cache_url=None): upload_dir_url=None, upload_cache_url=None,
signature_public_file=None, signature_private_file=None):
installer = Installer( installer = Installer(
dest, links, index, executable, always_unzip, path, newest, dest, links, index, executable, always_unzip, path, newest,
versions, use_dependency_links, allow_hosts=allow_hosts, versions, use_dependency_links, allow_hosts=allow_hosts,
...@@ -1112,7 +1119,9 @@ def install(specs, dest, ...@@ -1112,7 +1119,9 @@ def install(specs, dest,
allowed_eggs_from_site_packages=allowed_eggs_from_site_packages, allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
prefer_final=prefer_final, download_dir_url=download_dir_url, prefer_final=prefer_final, download_dir_url=download_dir_url,
download_cache_url=download_cache_url, upload_dir_url=upload_dir_url, download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
upload_cache_url=upload_cache_url) upload_cache_url=upload_cache_url,
signature_public_file=signature_public_file,
signature_private_file=signature_private_file)
return installer.install(specs, working_set) return installer.install(specs, working_set)
......
...@@ -50,7 +50,8 @@ def get_directory_key(url): ...@@ -50,7 +50,8 @@ def get_directory_key(url):
return 'slapos-buildout-%s' % urlmd5 return 'slapos-buildout-%s' % urlmd5
def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None): def download_network_cached(dir_url, cache_url, path, url, logger,
signature_public_file, signature_private_file, md5sum=None):
"""Downloads from a network cache provider """Downloads from a network cache provider
If something fail (providor be offline, or hash_string fail), we ignore If something fail (providor be offline, or hash_string fail), we ignore
...@@ -72,7 +73,10 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None): ...@@ -72,7 +73,10 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
logger.info('Downloading %s from network cache.' % url) logger.info('Downloading %s from network cache.' % url)
try: try:
nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url) nc = NetworkcacheClient(shacache=cache_url,
shadir=dir_url,
signature_private_file=signature_private_file,
signature_public_file=signature_public_file)
file_descriptor = nc.select(directory_key) file_descriptor = nc.select(directory_key)
buffer_size = min(1024, os.path.getsize(file_descriptor.name)) buffer_size = min(1024, os.path.getsize(file_descriptor.name))
...@@ -94,7 +98,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None): ...@@ -94,7 +98,8 @@ def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
return True return True
def upload_network_cached(dir_url, cache_url, external_url, path, logger): def upload_network_cached(dir_url, cache_url, external_url, path, logger,
signature_public_file, signature_private_file):
"""Upload file to a network cache server""" """Upload file to a network cache server"""
if not LIBNETWORKCACHE_ENABLED: if not LIBNETWORKCACHE_ENABLED:
return False return False
...@@ -112,7 +117,9 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger): ...@@ -112,7 +117,9 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger):
f = open(path, 'r') f = open(path, 'r')
try: try:
nc = NetworkcacheClient(shacache=cache_url, nc = NetworkcacheClient(shacache=cache_url,
shadir=dir_url) shadir=dir_url,
signature_public_file=signature_public_file,
signature_private_file=signature_private_file)
return nc.upload(f, directory_key, **kw) return nc.upload(f, directory_key, **kw)
except (IOError, UploadError), e: except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \ logger.info('Fail to upload file. %s' % \
......
...@@ -92,7 +92,9 @@ class Eggs(object): ...@@ -92,7 +92,9 @@ class Eggs(object):
('__networkcache__download-cache-url', 'download_cache_url'), ('__networkcache__download-cache-url', 'download_cache_url'),
('__networkcache__download-dir-url', 'download_dir_url'), ('__networkcache__download-dir-url', 'download_dir_url'),
('__networkcache__upload-cache-url', 'upload_cache_url'), ('__networkcache__upload-cache-url', 'upload_cache_url'),
('__networkcache__upload-dir-url', 'upload_dir_url')): ('__networkcache__upload-dir-url', 'upload_dir_url'),
('signature_private_file', 'signature_private_file'),
('signature_public_file', 'signature_public_file')):
if option_key in b_options: if option_key in b_options:
kw[kw_key] = b_options[option_key] kw[kw_key] = b_options[option_key]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment