Add network cache upload blacklist.

parent 144358f4
......@@ -160,8 +160,13 @@ signature-certificate-list =
5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
-----END CERTIFICATE-----
# List of URL(s) which shouldn't be installed from binary cache, separated by
# List of URL(s) which shouldn't be download from binary cache, separated by
# commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
binary-cache-url-blacklist =
download-from-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
# List of URL(s) which shouldn't be upload to binary cache, separated by
# commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
upload-to-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
......@@ -55,7 +55,8 @@ class Software(object):
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None,
binary_cache_url_blacklist = []):
download_from_binary_cache_url_blacklist = [],
upload_to_binary_cache_url_blacklist = []):
"""Initialisation of class parameters
"""
self.url = url
......@@ -78,7 +79,10 @@ class Software(object):
self.upload_binary_cache_url = upload_binary_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
self.binary_cache_url_blacklist = binary_cache_url_blacklist
self.download_from_binary_cache_url_blacklist = \
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
def install(self):
""" Fetches binary cache if possible.
......@@ -97,7 +101,7 @@ class Software(object):
self.software_url_hash,
tarpath, self.logger,
self.signature_certificate_list,
self.binary_cache_url_blacklist):
self.download_from_binary_cache_url_blacklist):
tar = tarfile.open(tarpath)
try:
self.logger.info("Extracting archive of cached software release...")
......@@ -106,9 +110,18 @@ class Software(object):
tar.close()
else:
self._install_from_buildout()
# Upload to binary cache if possible
blacklisted = False
for url in self.upload_to_binary_cache_url_blacklist:
if self.url.startswith(url):
blacklisted = True
self.logger.debug("Can't download from binary cache: "
"Software Release URL is blacklisted.")
if (self.software_root and self.url and self.software_url_hash \
and self.upload_binary_cache_url \
and self.upload_binary_dir_url):
and self.upload_binary_dir_url \
and not blacklisted):
self.logger.info("Creating archive of software release...")
tar = tarfile.open(tarpath, "w:gz")
try:
......
......@@ -50,7 +50,7 @@ def fallback_call(function):
@fallback_call
def download_network_cached(cache_url, dir_url, software_url, software_root,
key, path, logger, signature_certificate_list,
binary_cache_url_blacklist=None):
download_from_binary_cache_url_blacklist=None):
"""Downloads from a network cache provider
return True if download succeeded.
......@@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
if not(cache_url and dir_url and software_url and software_root):
return False
for url in binary_cache_url_blacklist:
for url in download_from_binary_cache_url_blacklist:
if software_url.startswith(url):
return False
......
......@@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
else:
signature_certificate_list = None
# Parse cache / binary options
option_dict["binary-cache-url-blacklist"] = [
url.strip() for url in option_dict.get("binary-cache-url-blacklist", ""
).split('\n') if url]
# Parse cache / binary cache options
# Backward compatibility about "binary-cache-url-blacklist" deprecated option
if option_dict.get("binary-cache-url-blacklist") and not \
option_dict.get("download-from-binary-cache-url-blacklist"):
option_dict["download-from-binary-cache-url-blacklist"] = \
option_dict["binary-cache-url-blacklist"]
option_dict["download-from-binary-cache-url-blacklist"] = [
url.strip() for url in option_dict.get(
"download-from-binary-cache-url-blacklist", "").split('\n') if url]
option_dict["upload-to-binary-cache-url-blacklist"] = [
url.strip() for url in option_dict.get(
"upload-to-binary-cache-url-blacklist", "").split('\n') if url]
# Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
# SlapOS Nodes configured with cron.
......@@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
option_dict.get('download-binary-cache-url', None),
upload_binary_cache_url=\
option_dict.get('upload-binary-cache-url', None),
binary_cache_url_blacklist=\
option_dict.get('binary-cache-url-blacklist', []),
download_from_binary_cache_url_blacklist=\
option_dict.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist=\
option_dict.get('upload-to-binary-cache-url-blacklist', []),
upload_cache_url=option_dict.get('upload-cache-url', None),
download_binary_dir_url=\
option_dict.get('download-binary-dir-url', None),
......@@ -370,7 +379,8 @@ class Slapgrid(object):
signature_certificate_list=None,
download_binary_cache_url=None,
upload_binary_cache_url=None,
binary_cache_url_blacklist=None,
download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None,
upload_cache_url=None,
download_binary_dir_url=None,
upload_binary_dir_url=None,
......@@ -403,7 +413,10 @@ class Slapgrid(object):
self.signature_certificate_list = signature_certificate_list
self.download_binary_cache_url = download_binary_cache_url
self.upload_binary_cache_url = upload_binary_cache_url
self.binary_cache_url_blacklist = binary_cache_url_blacklist
self.download_from_binary_cache_url_blacklist = \
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.upload_cache_url = upload_cache_url
self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url
......@@ -505,7 +518,10 @@ class Slapgrid(object):
signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url,
binary_cache_url_blacklist=self.binary_cache_url_blacklist,
download_from_binary_cache_url_blacklist=\
self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=\
self.upload_to_binary_cache_url_blacklist,
upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment