Add network cache upload blacklist.

parent 144358f4
...@@ -160,8 +160,13 @@ signature-certificate-list = ...@@ -160,8 +160,13 @@ signature-certificate-list =
5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e 5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI= x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
-----END CERTIFICATE----- -----END CERTIFICATE-----
# List of URL(s) which shouldn't be installed from binary cache, separated by # List of URL(s) which shouldn't be download from binary cache, separated by
# commas. Any URL beginning by a blacklisted URL will be blacklisted as well. # commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
binary-cache-url-blacklist = download-from-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
# List of URL(s) which shouldn't be upload to binary cache, separated by
# commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
upload-to-binary-cache-url-blacklist =
http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
...@@ -55,7 +55,8 @@ class Software(object): ...@@ -55,7 +55,8 @@ class Software(object):
shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None, shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
download_binary_cache_url=None, upload_binary_cache_url=None, download_binary_cache_url=None, upload_binary_cache_url=None,
download_binary_dir_url=None, upload_binary_dir_url=None, download_binary_dir_url=None, upload_binary_dir_url=None,
binary_cache_url_blacklist = []): download_from_binary_cache_url_blacklist = [],
upload_to_binary_cache_url_blacklist = []):
"""Initialisation of class parameters """Initialisation of class parameters
""" """
self.url = url self.url = url
...@@ -78,7 +79,10 @@ class Software(object): ...@@ -78,7 +79,10 @@ class Software(object):
self.upload_binary_cache_url = upload_binary_cache_url self.upload_binary_cache_url = upload_binary_cache_url
self.download_binary_dir_url = download_binary_dir_url self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url
self.binary_cache_url_blacklist = binary_cache_url_blacklist self.download_from_binary_cache_url_blacklist = \
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
def install(self): def install(self):
""" Fetches binary cache if possible. """ Fetches binary cache if possible.
...@@ -97,7 +101,7 @@ class Software(object): ...@@ -97,7 +101,7 @@ class Software(object):
self.software_url_hash, self.software_url_hash,
tarpath, self.logger, tarpath, self.logger,
self.signature_certificate_list, self.signature_certificate_list,
self.binary_cache_url_blacklist): self.download_from_binary_cache_url_blacklist):
tar = tarfile.open(tarpath) tar = tarfile.open(tarpath)
try: try:
self.logger.info("Extracting archive of cached software release...") self.logger.info("Extracting archive of cached software release...")
...@@ -106,9 +110,18 @@ class Software(object): ...@@ -106,9 +110,18 @@ class Software(object):
tar.close() tar.close()
else: else:
self._install_from_buildout() self._install_from_buildout()
# Upload to binary cache if possible
blacklisted = False
for url in self.upload_to_binary_cache_url_blacklist:
if self.url.startswith(url):
blacklisted = True
self.logger.debug("Can't download from binary cache: "
"Software Release URL is blacklisted.")
if (self.software_root and self.url and self.software_url_hash \ if (self.software_root and self.url and self.software_url_hash \
and self.upload_binary_cache_url \ and self.upload_binary_cache_url \
and self.upload_binary_dir_url): and self.upload_binary_dir_url \
and not blacklisted):
self.logger.info("Creating archive of software release...") self.logger.info("Creating archive of software release...")
tar = tarfile.open(tarpath, "w:gz") tar = tarfile.open(tarpath, "w:gz")
try: try:
......
...@@ -50,7 +50,7 @@ def fallback_call(function): ...@@ -50,7 +50,7 @@ def fallback_call(function):
@fallback_call @fallback_call
def download_network_cached(cache_url, dir_url, software_url, software_root, def download_network_cached(cache_url, dir_url, software_url, software_root,
key, path, logger, signature_certificate_list, key, path, logger, signature_certificate_list,
binary_cache_url_blacklist=None): download_from_binary_cache_url_blacklist=None):
"""Downloads from a network cache provider """Downloads from a network cache provider
return True if download succeeded. return True if download succeeded.
...@@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root, ...@@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
if not(cache_url and dir_url and software_url and software_root): if not(cache_url and dir_url and software_url and software_root):
return False return False
for url in binary_cache_url_blacklist: for url in download_from_binary_cache_url_blacklist:
if software_url.startswith(url): if software_url.startswith(url):
return False return False
......
...@@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): ...@@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
else: else:
signature_certificate_list = None signature_certificate_list = None
# Parse cache / binary cache options
# Parse cache / binary options # Backward compatibility about "binary-cache-url-blacklist" deprecated option
option_dict["binary-cache-url-blacklist"] = [ if option_dict.get("binary-cache-url-blacklist") and not \
url.strip() for url in option_dict.get("binary-cache-url-blacklist", "" option_dict.get("download-from-binary-cache-url-blacklist"):
).split('\n') if url] option_dict["download-from-binary-cache-url-blacklist"] = \
option_dict["binary-cache-url-blacklist"]
option_dict["download-from-binary-cache-url-blacklist"] = [
url.strip() for url in option_dict.get(
"download-from-binary-cache-url-blacklist", "").split('\n') if url]
option_dict["upload-to-binary-cache-url-blacklist"] = [
url.strip() for url in option_dict.get(
"upload-to-binary-cache-url-blacklist", "").split('\n') if url]
# Sleep for a random time to avoid SlapOS Master being DDOSed by an army of # Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
# SlapOS Nodes configured with cron. # SlapOS Nodes configured with cron.
...@@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): ...@@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
option_dict.get('download-binary-cache-url', None), option_dict.get('download-binary-cache-url', None),
upload_binary_cache_url=\ upload_binary_cache_url=\
option_dict.get('upload-binary-cache-url', None), option_dict.get('upload-binary-cache-url', None),
binary_cache_url_blacklist=\ download_from_binary_cache_url_blacklist=\
option_dict.get('binary-cache-url-blacklist', []), option_dict.get('download-from-binary-cache-url-blacklist', []),
upload_to_binary_cache_url_blacklist=\
option_dict.get('upload-to-binary-cache-url-blacklist', []),
upload_cache_url=option_dict.get('upload-cache-url', None), upload_cache_url=option_dict.get('upload-cache-url', None),
download_binary_dir_url=\ download_binary_dir_url=\
option_dict.get('download-binary-dir-url', None), option_dict.get('download-binary-dir-url', None),
...@@ -370,7 +379,8 @@ class Slapgrid(object): ...@@ -370,7 +379,8 @@ class Slapgrid(object):
signature_certificate_list=None, signature_certificate_list=None,
download_binary_cache_url=None, download_binary_cache_url=None,
upload_binary_cache_url=None, upload_binary_cache_url=None,
binary_cache_url_blacklist=None, download_from_binary_cache_url_blacklist=None,
upload_to_binary_cache_url_blacklist=None,
upload_cache_url=None, upload_cache_url=None,
download_binary_dir_url=None, download_binary_dir_url=None,
upload_binary_dir_url=None, upload_binary_dir_url=None,
...@@ -403,7 +413,10 @@ class Slapgrid(object): ...@@ -403,7 +413,10 @@ class Slapgrid(object):
self.signature_certificate_list = signature_certificate_list self.signature_certificate_list = signature_certificate_list
self.download_binary_cache_url = download_binary_cache_url self.download_binary_cache_url = download_binary_cache_url
self.upload_binary_cache_url = upload_binary_cache_url self.upload_binary_cache_url = upload_binary_cache_url
self.binary_cache_url_blacklist = binary_cache_url_blacklist self.download_from_binary_cache_url_blacklist = \
download_from_binary_cache_url_blacklist
self.upload_to_binary_cache_url_blacklist = \
upload_to_binary_cache_url_blacklist
self.upload_cache_url = upload_cache_url self.upload_cache_url = upload_cache_url
self.download_binary_dir_url = download_binary_dir_url self.download_binary_dir_url = download_binary_dir_url
self.upload_binary_dir_url = upload_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url
...@@ -505,7 +518,10 @@ class Slapgrid(object): ...@@ -505,7 +518,10 @@ class Slapgrid(object):
signature_certificate_list=self.signature_certificate_list, signature_certificate_list=self.signature_certificate_list,
download_binary_cache_url=self.download_binary_cache_url, download_binary_cache_url=self.download_binary_cache_url,
upload_binary_cache_url=self.upload_binary_cache_url, upload_binary_cache_url=self.upload_binary_cache_url,
binary_cache_url_blacklist=self.binary_cache_url_blacklist, download_from_binary_cache_url_blacklist=\
self.download_from_binary_cache_url_blacklist,
upload_to_binary_cache_url_blacklist=\
self.upload_to_binary_cache_url_blacklist,
upload_cache_url=self.upload_cache_url, upload_cache_url=self.upload_cache_url,
download_binary_dir_url=self.download_binary_dir_url, download_binary_dir_url=self.download_binary_dir_url,
upload_binary_dir_url=self.upload_binary_dir_url, upload_binary_dir_url=self.upload_binary_dir_url,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment