Commit a58f5b20 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Revert "Revert "Remove NetworkCache class in favor of standard NetworkcacheClient class""

This reverts commit 32409436.

We can now safely reapply abba1a33 since
all machines are using http URL. Indeed, the playbook was updated a long
time ago in slapos.package@fa5fe656
parent ab549297
...@@ -40,110 +40,6 @@ from random import choice ...@@ -40,110 +40,6 @@ from random import choice
from string import ascii_lowercase from string import ascii_lowercase
from slapos.libnetworkcache import NetworkcacheClient from slapos.libnetworkcache import NetworkcacheClient
from slapos.networkcachehelper import helper_download_network_cached
class NetworkCache:
def __init__(self, configuration_path):
if not os.path.exists(configuration_path):
raise ValueError("You need configuration file")
self.configuration = configuration_path
self._load()
def _load(self):
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.configuration)
network_cache_info_dict = dict(network_cache_info.items('networkcache'))
def get_(name):
return network_cache_info_dict.get(name)
self.download_binary_cache_url = get_('download-binary-cache-url')
self.download_cache_url = get_('download-cache-url')
self.download_binary_dir_url = get_('download-binary-dir-url')
self.signature_certificate_list = get_('signature-certificate-list')
# Not mandatory
self.dir_url = get_('upload-dir-url')
self.cache_url = get_('upload-cache-url')
key_file_key = 'signature-private-key-file'
self.signature_private_key_file = get_(key_file_key)
key_file_old_key = 'signature_private_key_file'
if key_file_old_key in network_cache_info_dict:
raise ValueError(
'%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
self.shacache_ca_file = get_('shacache-ca-file')
self.shacache_cert_file = get_('shacache-cert-file')
self.shacache_key_file = get_('shacache-key-file')
self.shadir_cert_file = get_('shadir-cert-file')
self.shadir_key_file = get_('shadir-key-file')
self.shadir_ca_file = get_('shadir-ca-file')
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def upload(self, path, metadata_dict, is_sha256file=False):
"""
Upload an existing file, using a file_descriptor.
"""
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
file_descriptor = open(path, 'rb')
if not (self.dir_url and self.cache_url):
raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
# backward compatibility
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
with NetworkcacheClient(self.cache_url, self.dir_url,
signature_private_key_file=self.signature_private_key_file or None,
signature_certificate_list=self.signature_certificate_list or None,
shacache_ca_file=self.shacache_ca_file or None,
shacache_cert_file=self.shacache_cert_file or None,
shacache_key_file=self.shacache_key_file or None,
shadir_cert_file=self.shadir_cert_file or None,
shadir_key_file=self.shadir_key_file or None,
shadir_ca_file=self.shadir_ca_file or None,
) as nc:
return nc.upload(file_descriptor, key, **metadata_dict)
def download(self, path, wanted_metadata_dict={},
required_key_list=[], strategy=None, is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
result = helper_download_network_cached(
self.download_binary_dir_url,
self.download_binary_cache_url,
self.signature_certificate_list,
key, wanted_metadata_dict,
required_key_list, strategy)
if result:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
file_descriptor, metadata_dict = result
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
# XXX method should check MD5.
return metadata_dict
finally:
f.close()
file_descriptor.close()
return False
def strategy(entry_list): def strategy(entry_list):
...@@ -162,6 +58,13 @@ class Signature: ...@@ -162,6 +58,13 @@ class Signature:
def __init__(self, config, logger=None): def __init__(self, config, logger=None):
self.config = config self.config = config
self.logger = logger self.logger = logger
self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.config.slapos_configuration)
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def log(self, message, *args): def log(self, message, *args):
if self.logger is not None: if self.logger is not None:
...@@ -183,28 +86,62 @@ class Signature: ...@@ -183,28 +86,62 @@ class Signature:
with open(destination, "wb") as f: with open(destination, "wb") as f:
f.write(base) f.write(base)
def _download_once(self, path, wanted_metadata_dict={},
required_key_list=[], is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
self.log('Downloading %s...', key)
result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
entry = None
result = list(result)
if result:
entry = strategy(result)
if not entry: # XXX: this should be the choice of 'strategy' function
self.log("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.")
entry = result[0]
if not entry:
self.log('No entry matching key %s', key)
else:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
f = open(path, 'w+b')
fd_download = self.shacache.download(entry['sha512'])
try:
shutil.copyfileobj(fd_download, f)
# XXX method should check MD5.
return entry
finally:
f.close()
fd_download.close()
return False
def _download(self, path): def _download(self, path):
""" """
Download a tar of the repository from cache, and untar it. Download a tar of the repository from cache, and untar it.
""" """
shacache = NetworkCache(self.config.slapos_configuration) try:
download_metadata_dict = self._download_once(path=path,
if shacache.signature_certificate_list is None: required_key_list=['timestamp'])
raise ValueError("You need at least one valid signature for download") except:
return False
download_metadata_dict = shacache.download(path=path,
required_key_list=['timestamp'], strategy=strategy)
if download_metadata_dict: if download_metadata_dict:
self.log('File downloaded in %s', path) self.log('File downloaded in %s', path)
current_sha256 = self.get_file_hash(path) current_sha256 = self.get_file_hash(path)
with tempfile.NamedTemporaryFile() as f_256: with tempfile.NamedTemporaryFile() as f_256:
sha256path = f_256.name sha256path = f_256.name
if shacache.download(path=sha256path, required_key_list=['timestamp'], try:
strategy=strategy, is_sha256file=True): sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
self.log('sha 256 downloaded in %s', sha256path) self.log('sha 256 downloaded in %s', sha256path)
expected_sha256 = f_256.read() except:
sha256sum_present = False
if sha256sum_present:
expected_sha256 = f_256.read()
if current_sha256 == expected_sha256: if current_sha256 == expected_sha256:
return True return True
else: else:
...@@ -230,8 +167,6 @@ class Signature: ...@@ -230,8 +167,6 @@ class Signature:
""" """
Creates uploads repository to cache. Creates uploads repository to cache.
""" """
shacache = NetworkCache(self.config.slapos_configuration)
sha256path = path + ".sha256" sha256path = path + ".sha256"
self.save_file_hash(path, sha256path) self.save_file_hash(path, sha256path)
...@@ -239,20 +174,25 @@ class Signature: ...@@ -239,20 +174,25 @@ class Signature:
# XXX: we set date from client side. It can be potentially dangerous # XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured. # as it can be badly configured.
'timestamp': time.time(), 'timestamp': time.time(),
'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) 'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
# backward compatibility
'file': 'notused',
'urlmd5': 'notused'
} }
try: try:
sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict) sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
if sha512sum: if sha512sum:
self.log( self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', path, 'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
shacache.key, sha512sum) self.key, sha512sum)
sha512sum_path = shacache.upload( sha512sum_path = self.shacache.upload(
path=sha256path, metadata_dict=metadata_dict, is_sha256file=True) open(sha256path, 'rb'),
self.key + "-sha256-content",
**metadata_dict)
if sha512sum_path: if sha512sum_path:
self.log( self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path, 'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
shacache.key, sha512sum_path) self.key, sha512sum_path)
else: else:
self.log('Fail to upload sha256file file to cache.') self.log('Fail to upload sha256file file to cache.')
else: else:
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment