Revert "Remove NetworkCache class in favor of standard NetworkcacheClient class"

This reverts commit abba1a33.

We revert temporarily this commit until the slapcache.cfg file has been
updated on all machines (remove https from download-cache-url)

slapcache/signature.py

@@ -40,6 +40,110 @@ from random import choice
 from string import ascii_lowercase
 
 from slapos.libnetworkcache import NetworkcacheClient
+from slapos.networkcachehelper import helper_download_network_cached
+
+
+class NetworkCache:
+  def __init__(self, configuration_path):
+    if not os.path.exists(configuration_path):
+      raise ValueError("You need configuration file")
+    self.configuration = configuration_path
+    self._load()
+ 
+  def _load(self):
+    network_cache_info = configparser.RawConfigParser()
+    network_cache_info.read(self.configuration)
+    network_cache_info_dict = dict(network_cache_info.items('networkcache'))
+    def get_(name):
+      return network_cache_info_dict.get(name)
+
+    self.download_binary_cache_url = get_('download-binary-cache-url')
+    self.download_cache_url = get_('download-cache-url')
+    self.download_binary_dir_url = get_('download-binary-dir-url')
+    self.signature_certificate_list = get_('signature-certificate-list')
+
+    # Not mandatory
+    self.dir_url = get_('upload-dir-url')
+    self.cache_url = get_('upload-cache-url')
+    key_file_key = 'signature-private-key-file'
+    self.signature_private_key_file = get_(key_file_key)
+    key_file_old_key = 'signature_private_key_file'
+    if key_file_old_key in network_cache_info_dict:
+      raise ValueError(
+        '%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
+    self.shacache_ca_file = get_('shacache-ca-file')
+    self.shacache_cert_file = get_('shacache-cert-file')
+    self.shacache_key_file = get_('shacache-key-file')
+    self.shadir_cert_file = get_('shadir-cert-file')
+    self.shadir_key_file = get_('shadir-key-file')
+    self.shadir_ca_file = get_('shadir-ca-file')
+
+    if network_cache_info.has_section('shacache'):
+      self.key = network_cache_info.get('shacache', 'key')
+    else:
+      self.key = "slapos-upgrade-testing-key"
+
+  def upload(self, path, metadata_dict, is_sha256file=False):
+    """
+    Upload an existing file, using a file_descriptor.
+    """
+    if is_sha256file: 
+      key = self.key + "-sha256-content"
+    else:
+      key = self.key
+
+    file_descriptor = open(path, 'rb')
+    if not (self.dir_url and self.cache_url):
+      raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
+
+    # backward compatibility
+    metadata_dict.setdefault('file', 'notused')
+    metadata_dict.setdefault('urlmd5', 'notused')
+
+    # convert '' into None in order to call nc nicely
+    with NetworkcacheClient(self.cache_url, self.dir_url,
+          signature_private_key_file=self.signature_private_key_file or None,
+          signature_certificate_list=self.signature_certificate_list or None,
+          shacache_ca_file=self.shacache_ca_file or None,
+          shacache_cert_file=self.shacache_cert_file or None,
+          shacache_key_file=self.shacache_key_file or None,
+          shadir_cert_file=self.shadir_cert_file or None,
+          shadir_key_file=self.shadir_key_file or None,
+          shadir_ca_file=self.shadir_ca_file or None,
+
+        ) as nc:
+      return nc.upload(file_descriptor, key, **metadata_dict)
+
+
+  def download(self, path, wanted_metadata_dict={}, 
+                 required_key_list=[], strategy=None, is_sha256file=False):
+
+    if is_sha256file:
+      key = self.key + "-sha256-content"
+    else:
+      key = self.key
+
+    result = helper_download_network_cached(
+              self.download_binary_dir_url, 
+              self.download_binary_cache_url,
+              self.signature_certificate_list,
+              key, wanted_metadata_dict, 
+              required_key_list, strategy)
+
+    if result:
+      # XXX check if nc filters signature_certificate_list!
+      # Creates a file with content to desired path.
+      file_descriptor, metadata_dict = result
+      f = open(path, 'w+b')
+      try:
+        shutil.copyfileobj(file_descriptor, f)
+        # XXX method should check MD5.
+        return metadata_dict
+      finally:
+        f.close()
+        file_descriptor.close()
+    return False
+
 
 
 def strategy(entry_list):
@@ -51,20 +155,13 @@ def strategy(entry_list):
       best_entry = entry
       timestamp = entry['timestamp']
 
-  return best_entry
+  return best_entry 
 
 class Signature:
 
   def __init__(self, config, logger=None):
     self.config = config
     self.logger = logger
-    self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
-    network_cache_info = configparser.RawConfigParser()
-    network_cache_info.read(self.config.slapos_configuration)
-    if network_cache_info.has_section('shacache'):
-      self.key = network_cache_info.get('shacache', 'key')
-    else:
-      self.key = "slapos-upgrade-testing-key"
 
   def log(self, message, *args):
     if self.logger is not None:
@@ -82,66 +179,32 @@ class Signature:
     return base
 
   def save_file_hash(self, path, destination):
-    base = self.get_file_hash(path)
+    base = self.get_file_hash(path) 
     with open(destination, "wb") as f:
       f.write(base)
 
-  def _download_once(self, path, wanted_metadata_dict={},
-                 required_key_list=[], is_sha256file=False):
-
-    if is_sha256file:
-      key = self.key + "-sha256-content"
-    else:
-      key = self.key
-
-    self.log('Downloading %s...', key)
-    result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
-    entry = None
-    result = list(result)
-    if result:
-      entry = strategy(result)
-      if not entry: # XXX: this should be the choice of 'strategy' function
-        self.log("Can't find best entry matching strategy, selecting "
-            "random one between acceptable ones.")
-        entry = result[0]
-    if not entry:
-      self.log('No entry matching key %s', key)
-    else:
-      # XXX check if nc filters signature_certificate_list!
-      # Creates a file with content to desired path.
-      f = open(path, 'w+b')
-      fd_download = self.shacache.download(entry['sha512'])
-      try:
-        shutil.copyfileobj(fd_download, f)
-        # XXX method should check MD5.
-        return entry
-      finally:
-        f.close()
-        fd_download.close()
-    return False
-
   def _download(self, path):
     """
     Download a tar of the repository from cache, and untar it.
     """
-    try:
-      download_metadata_dict = self._download_once(path=path,
-              required_key_list=['timestamp'])
-    except:
-      return False
+    shacache = NetworkCache(self.config.slapos_configuration)
+
+    if shacache.signature_certificate_list is None:
+      raise ValueError("You need at least one valid signature for download")
+
+    download_metadata_dict = shacache.download(path=path, 
+           required_key_list=['timestamp'], strategy=strategy)
+
     if download_metadata_dict:
       self.log('File downloaded in %s', path)
       current_sha256 = self.get_file_hash(path)
       with tempfile.NamedTemporaryFile() as f_256:
         sha256path = f_256.name
-        try:
-          sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
+        if shacache.download(path=sha256path, required_key_list=['timestamp'],
+                              strategy=strategy, is_sha256file=True):
           self.log('sha 256 downloaded in %s', sha256path)
-        except:
-          sha256sum_present = False
-
-        if sha256sum_present:
           expected_sha256 = f_256.read()
+
           if current_sha256 == expected_sha256:
             return True
           else:
@@ -167,32 +230,29 @@ class Signature:
     """
     Creates uploads repository to cache.
     """
+    shacache = NetworkCache(self.config.slapos_configuration)
+
     sha256path = path + ".sha256"
     self.save_file_hash(path, sha256path)
-
+ 
     metadata_dict = {
       # XXX: we set date from client side. It can be potentially dangerous
       # as it can be badly configured.
       'timestamp': time.time(),
-      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
-      # backward compatibility
-      'file': 'notused',
-      'urlmd5': 'notused'
+      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) 
     }
     try:
-      sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
+      sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict)
       if sha512sum:
         self.log(
           'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
-          self.key, sha512sum)
-        sha512sum_path = self.shacache.upload(
-            open(sha256path, 'rb'),
-            self.key + "-sha256-content",
-            **metadata_dict)
+          shacache.key, sha512sum)
+        sha512sum_path = shacache.upload(
+          path=sha256path, metadata_dict=metadata_dict, is_sha256file=True)
         if sha512sum_path:
           self.log(
             'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
-            self.key, sha512sum_path)
+            shacache.key, sha512sum_path)
         else:
           self.log('Fail to upload sha256file file to cache.')
       else:
@@ -203,7 +263,7 @@ class Signature:
 
   def upload(self):
     self._upload(self.config.file)
-
+ 
 # Class containing all parameters needed for configuration
 class Config:
   def __init__(self, option_dict=None):

[Romain Courteaud]

@tomo this introduced an error in the test

ModuleNotFoundError: No module named 'slapos.networkcachehelper'

[Thomas Gambier]

Yes, I know, it works outside of tests because the commit slapos.libnetworkcache@5c59ccb1 was not pushed to pypi.

I plan to re-revert this this week and the error will be gone.

Sorry for the inconvenience.

[Jérome Perrin]

@tomo it seems we still have an error here, is it something known ?

[Thomas Gambier]

Sorry, I forgot to do it. I just did now in a58f5b20