Commit 63b17738 by Łukasz Nowak

slapgrid: binary cache only option

/reviewed-on !173
1 parent 78f01daf
Pipeline #7389 for 63b17738 failed in 0 seconds
......@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
/
# List of URL(s) which are forced to be downloaded from binary cache
#download-from-binary-cache-force-url-list =
# https://lab.node.vifib.com/nexedi/slapos/raw/1.0.
......@@ -112,6 +112,7 @@ class Software(object):
download_binary_dir_url=None, upload_binary_dir_url=None,
download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None,
download_from_binary_cache_force_url_list=None,
software_min_free_space=None,
buildout_debug=False,
shared_part_list=''):
......@@ -124,6 +125,9 @@ class Software(object):
if upload_to_binary_cache_url_blacklist is None:
upload_to_binary_cache_url_blacklist = []
if download_from_binary_cache_force_url_list is None:
download_from_binary_cache_force_url_list = []
self.url = url
self.software_root = software_root
self.software_url_hash = md5digest(self.url)
......@@ -151,6 +155,8 @@ class Software(object):
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.download_from_binary_cache_force_url_list = \
download_from_binary_cache_force_url_list
self.software_min_free_space = software_min_free_space
def check_free_space(self):
......@@ -175,6 +181,12 @@ class Software(object):
try:
tarpath = os.path.join(cache_dir, self.software_url_hash)
# Check if we can download from cache
force_binary_cache = False
for url_match in self.download_from_binary_cache_force_url_list:
if self.url.startswith(url_match):
force_binary_cache = True
self.logger.debug('Binary cache forced for %r because of match %r', self.url, url_match)
break
if (not os.path.exists(self.software_path)) \
and download_network_cached(
self.download_binary_cache_url,
......@@ -190,6 +202,10 @@ class Software(object):
tar.extractall(path=self.software_root)
finally:
tar.close()
elif force_binary_cache:
message = 'Binary cache forced for %r, but failed to download, will retry again' % (self.url,)
self.logger.error(message)
raise BuildoutFailedError(message)
else:
self._install_from_buildout()
# Upload to binary cache if possible and allowed
......
......@@ -188,6 +188,9 @@ def merged_options(args, configp):
options["upload-to-binary-cache-url-blacklist"] = [
url.strip() for url in options.get(
"upload-to-binary-cache-url-blacklist", "").split('\n') if url]
options["download-from-binary-cache-force-url-list"] = [
url.strip() for url in options.get(
"download-from-binary-cache-force-url-list", "").split('\n') if url]
options['firewall'] = {}
if configp.has_section('firewall'):
......@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
op.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist=
op.get('upload-to-binary-cache-url-blacklist', []),
download_from_binary_cache_force_url_list=
op.get('download-from-binary-cache-force-url-list', []),
upload_cache_url=op.get('upload-cache-url'),
download_binary_dir_url=op.get('download-binary-dir-url'),
upload_binary_dir_url=op.get('upload-binary-dir-url'),
......@@ -317,6 +322,7 @@ class Slapgrid(object):
upload_binary_cache_url=None,
download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None,
download_from_binary_cache_force_url_list=None,
upload_cache_url=None,
download_binary_dir_url=None,
upload_binary_dir_url=None,
......@@ -362,6 +368,8 @@ class Slapgrid(object):
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.download_from_binary_cache_force_url_list = \
download_from_binary_cache_force_url_list
self.upload_cache_url = upload_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
......@@ -555,6 +563,8 @@ stderr_logfile_backups=1
self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=
self.upload_to_binary_cache_url_blacklist,
download_from_binary_cache_force_url_list=
self.download_from_binary_cache_force_url_list,
upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url,
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!