Commit abba1a33 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Remove NetworkCache class in favor of standard NetworkcacheClient class

WARNING: this fixes a very old bug that download-binary-dir-url was used
in place of download-dir-url. This may break existing instance of
slapcache.
parent 9a8c10e5
......@@ -40,110 +40,6 @@ from random import choice
from string import ascii_lowercase
from slapos.libnetworkcache import NetworkcacheClient
from slapos.networkcachehelper import helper_download_network_cached
class NetworkCache:
def __init__(self, configuration_path):
if not os.path.exists(configuration_path):
raise ValueError("You need configuration file")
self.configuration = configuration_path
self._load()
def _load(self):
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.configuration)
network_cache_info_dict = dict(network_cache_info.items('networkcache'))
def get_(name):
return network_cache_info_dict.get(name)
self.download_binary_cache_url = get_('download-binary-cache-url')
self.download_cache_url = get_('download-cache-url')
self.download_binary_dir_url = get_('download-binary-dir-url')
self.signature_certificate_list = get_('signature-certificate-list')
# Not mandatory
self.dir_url = get_('upload-dir-url')
self.cache_url = get_('upload-cache-url')
key_file_key = 'signature-private-key-file'
self.signature_private_key_file = get_(key_file_key)
key_file_old_key = 'signature_private_key_file'
if key_file_old_key in network_cache_info_dict:
raise ValueError(
'%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
self.shacache_ca_file = get_('shacache-ca-file')
self.shacache_cert_file = get_('shacache-cert-file')
self.shacache_key_file = get_('shacache-key-file')
self.shadir_cert_file = get_('shadir-cert-file')
self.shadir_key_file = get_('shadir-key-file')
self.shadir_ca_file = get_('shadir-ca-file')
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def upload(self, path, metadata_dict, is_sha256file=False):
"""
Upload an existing file, using a file_descriptor.
"""
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
file_descriptor = open(path, 'rb')
if not (self.dir_url and self.cache_url):
raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
# backward compatibility
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
with NetworkcacheClient(self.cache_url, self.dir_url,
signature_private_key_file=self.signature_private_key_file or None,
signature_certificate_list=self.signature_certificate_list or None,
shacache_ca_file=self.shacache_ca_file or None,
shacache_cert_file=self.shacache_cert_file or None,
shacache_key_file=self.shacache_key_file or None,
shadir_cert_file=self.shadir_cert_file or None,
shadir_key_file=self.shadir_key_file or None,
shadir_ca_file=self.shadir_ca_file or None,
) as nc:
return nc.upload(file_descriptor, key, **metadata_dict)
def download(self, path, wanted_metadata_dict={},
required_key_list=[], strategy=None, is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
result = helper_download_network_cached(
self.download_binary_dir_url,
self.download_binary_cache_url,
self.signature_certificate_list,
key, wanted_metadata_dict,
required_key_list, strategy)
if result:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
file_descriptor, metadata_dict = result
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
# XXX method should check MD5.
return metadata_dict
finally:
f.close()
file_descriptor.close()
return False
def strategy(entry_list):
......@@ -162,6 +58,13 @@ class Signature:
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.config.slapos_configuration)
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def log(self, message, *args):
if self.logger is not None:
......@@ -183,28 +86,62 @@ class Signature:
with open(destination, "wb") as f:
f.write(base)
def _download_once(self, path, wanted_metadata_dict={},
required_key_list=[], is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
self.log('Downloading %s...', key)
result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
entry = None
result = list(result)
if result:
entry = strategy(result)
if not entry: # XXX: this should be the choice of 'strategy' function
self.log("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.")
entry = result[0]
if not entry:
self.log('No entry matching key %s', key)
else:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
f = open(path, 'w+b')
fd_download = self.shacache.download(entry['sha512'])
try:
shutil.copyfileobj(fd_download, f)
# XXX method should check MD5.
return entry
finally:
f.close()
fd_download.close()
return False
def _download(self, path):
"""
Download a tar of the repository from cache, and untar it.
"""
shacache = NetworkCache(self.config.slapos_configuration)
if shacache.signature_certificate_list is None:
raise ValueError("You need at least one valid signature for download")
download_metadata_dict = shacache.download(path=path,
required_key_list=['timestamp'], strategy=strategy)
try:
download_metadata_dict = self._download_once(path=path,
required_key_list=['timestamp'])
except:
return False
if download_metadata_dict:
self.log('File downloaded in %s', path)
current_sha256 = self.get_file_hash(path)
with tempfile.NamedTemporaryFile() as f_256:
sha256path = f_256.name
if shacache.download(path=sha256path, required_key_list=['timestamp'],
strategy=strategy, is_sha256file=True):
try:
sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
self.log('sha 256 downloaded in %s', sha256path)
expected_sha256 = f_256.read()
except:
sha256sum_present = False
if sha256sum_present:
expected_sha256 = f_256.read()
if current_sha256 == expected_sha256:
return True
else:
......@@ -230,8 +167,6 @@ class Signature:
"""
Creates uploads repository to cache.
"""
shacache = NetworkCache(self.config.slapos_configuration)
sha256path = path + ".sha256"
self.save_file_hash(path, sha256path)
......@@ -239,20 +174,25 @@ class Signature:
# XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured.
'timestamp': time.time(),
'token': ''.join([choice(ascii_lowercase) for _ in range(128)])
'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
# backward compatibility
'file': 'notused',
'urlmd5': 'notused'
}
try:
sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict)
sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
if sha512sum:
self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
shacache.key, sha512sum)
sha512sum_path = shacache.upload(
path=sha256path, metadata_dict=metadata_dict, is_sha256file=True)
self.key, sha512sum)
sha512sum_path = self.shacache.upload(
open(sha256path, 'rb'),
self.key + "-sha256-content",
**metadata_dict)
if sha512sum_path:
self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
shacache.key, sha512sum_path)
self.key, sha512sum_path)
else:
self.log('Fail to upload sha256file file to cache.')
else:
......
......@@ -5,6 +5,7 @@ key = %(key)s
download-binary-cache-url = http://shacache.nxdcdn.com
download-cache-url = https://shacache.nxdcdn.com
download-binary-dir-url = http://shadir.nxdcdn.com
download-dir-url = http://shadir.nxdcdn.com
# Standard Signature handled by SlapOS Administrators
signature-certificate-list =
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment