Commit 32409436 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Revert "Remove NetworkCache class in favor of standard NetworkcacheClient class"

This reverts commit abba1a33.

We revert temporarily this commit until the slapcache.cfg file has been
updated on all machines (remove https from download-cache-url)
parent 71aae4df
......@@ -40,6 +40,110 @@ from random import choice
from string import ascii_lowercase
from slapos.libnetworkcache import NetworkcacheClient
from slapos.networkcachehelper import helper_download_network_cached
  • @tomo this introduced an error in the test

    ModuleNotFoundError: No module named 'slapos.networkcachehelper'
  • Yes, I know, it works outside of tests because the commit slapos.libnetworkcache@5c59ccb1 was not pushed to pypi.

    I plan to re-revert this this week and the error will be gone.

    Sorry for the inconvenience.

  • @tomo it seems we still have an error here, is it something known ?

  • Sorry, I forgot to do it. I just did now in a58f5b20

Please register or sign in to reply
class NetworkCache:
def __init__(self, configuration_path):
if not os.path.exists(configuration_path):
raise ValueError("You need configuration file")
self.configuration = configuration_path
self._load()
def _load(self):
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.configuration)
network_cache_info_dict = dict(network_cache_info.items('networkcache'))
def get_(name):
return network_cache_info_dict.get(name)
self.download_binary_cache_url = get_('download-binary-cache-url')
self.download_cache_url = get_('download-cache-url')
self.download_binary_dir_url = get_('download-binary-dir-url')
self.signature_certificate_list = get_('signature-certificate-list')
# Not mandatory
self.dir_url = get_('upload-dir-url')
self.cache_url = get_('upload-cache-url')
key_file_key = 'signature-private-key-file'
self.signature_private_key_file = get_(key_file_key)
key_file_old_key = 'signature_private_key_file'
if key_file_old_key in network_cache_info_dict:
raise ValueError(
'%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
self.shacache_ca_file = get_('shacache-ca-file')
self.shacache_cert_file = get_('shacache-cert-file')
self.shacache_key_file = get_('shacache-key-file')
self.shadir_cert_file = get_('shadir-cert-file')
self.shadir_key_file = get_('shadir-key-file')
self.shadir_ca_file = get_('shadir-ca-file')
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def upload(self, path, metadata_dict, is_sha256file=False):
"""
Upload an existing file, using a file_descriptor.
"""
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
file_descriptor = open(path, 'rb')
if not (self.dir_url and self.cache_url):
raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
# backward compatibility
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
with NetworkcacheClient(self.cache_url, self.dir_url,
signature_private_key_file=self.signature_private_key_file or None,
signature_certificate_list=self.signature_certificate_list or None,
shacache_ca_file=self.shacache_ca_file or None,
shacache_cert_file=self.shacache_cert_file or None,
shacache_key_file=self.shacache_key_file or None,
shadir_cert_file=self.shadir_cert_file or None,
shadir_key_file=self.shadir_key_file or None,
shadir_ca_file=self.shadir_ca_file or None,
) as nc:
return nc.upload(file_descriptor, key, **metadata_dict)
def download(self, path, wanted_metadata_dict={},
required_key_list=[], strategy=None, is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
result = helper_download_network_cached(
self.download_binary_dir_url,
self.download_binary_cache_url,
self.signature_certificate_list,
key, wanted_metadata_dict,
required_key_list, strategy)
if result:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
file_descriptor, metadata_dict = result
f = open(path, 'w+b')
try:
shutil.copyfileobj(file_descriptor, f)
# XXX method should check MD5.
return metadata_dict
finally:
f.close()
file_descriptor.close()
return False
def strategy(entry_list):
......@@ -58,13 +162,6 @@ class Signature:
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
network_cache_info = configparser.RawConfigParser()
network_cache_info.read(self.config.slapos_configuration)
if network_cache_info.has_section('shacache'):
self.key = network_cache_info.get('shacache', 'key')
else:
self.key = "slapos-upgrade-testing-key"
def log(self, message, *args):
if self.logger is not None:
......@@ -86,62 +183,28 @@ class Signature:
with open(destination, "wb") as f:
f.write(base)
def _download_once(self, path, wanted_metadata_dict={},
required_key_list=[], is_sha256file=False):
if is_sha256file:
key = self.key + "-sha256-content"
else:
key = self.key
self.log('Downloading %s...', key)
result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
entry = None
result = list(result)
if result:
entry = strategy(result)
if not entry: # XXX: this should be the choice of 'strategy' function
self.log("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.")
entry = result[0]
if not entry:
self.log('No entry matching key %s', key)
else:
# XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path.
f = open(path, 'w+b')
fd_download = self.shacache.download(entry['sha512'])
try:
shutil.copyfileobj(fd_download, f)
# XXX method should check MD5.
return entry
finally:
f.close()
fd_download.close()
return False
def _download(self, path):
"""
Download a tar of the repository from cache, and untar it.
"""
try:
download_metadata_dict = self._download_once(path=path,
required_key_list=['timestamp'])
except:
return False
shacache = NetworkCache(self.config.slapos_configuration)
if shacache.signature_certificate_list is None:
raise ValueError("You need at least one valid signature for download")
download_metadata_dict = shacache.download(path=path,
required_key_list=['timestamp'], strategy=strategy)
if download_metadata_dict:
self.log('File downloaded in %s', path)
current_sha256 = self.get_file_hash(path)
with tempfile.NamedTemporaryFile() as f_256:
sha256path = f_256.name
try:
sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
if shacache.download(path=sha256path, required_key_list=['timestamp'],
strategy=strategy, is_sha256file=True):
self.log('sha 256 downloaded in %s', sha256path)
except:
sha256sum_present = False
if sha256sum_present:
expected_sha256 = f_256.read()
if current_sha256 == expected_sha256:
return True
else:
......@@ -167,6 +230,8 @@ class Signature:
"""
Creates uploads repository to cache.
"""
shacache = NetworkCache(self.config.slapos_configuration)
sha256path = path + ".sha256"
self.save_file_hash(path, sha256path)
......@@ -174,25 +239,20 @@ class Signature:
# XXX: we set date from client side. It can be potentially dangerous
# as it can be badly configured.
'timestamp': time.time(),
'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
# backward compatibility
'file': 'notused',
'urlmd5': 'notused'
'token': ''.join([choice(ascii_lowercase) for _ in range(128)])
}
try:
sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict)
if sha512sum:
self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
self.key, sha512sum)
sha512sum_path = self.shacache.upload(
open(sha256path, 'rb'),
self.key + "-sha256-content",
**metadata_dict)
shacache.key, sha512sum)
sha512sum_path = shacache.upload(
path=sha256path, metadata_dict=metadata_dict, is_sha256file=True)
if sha512sum_path:
self.log(
'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
self.key, sha512sum_path)
shacache.key, sha512sum_path)
else:
self.log('Fail to upload sha256file file to cache.')
else:
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment