Revert "Revert "Remove NetworkCache class in favor of standard NetworkcacheClient class""

This reverts commit 32409436.

We can now safely reapply abba1a33 since
all machines are using http URL. Indeed, the playbook was updated a long
time ago in slapos.package@fa5fe656

slapcache/signature.py

@@ -40,110 +40,6 @@ from random import choice
 from string import ascii_lowercase
 
 from slapos.libnetworkcache import NetworkcacheClient
-from slapos.networkcachehelper import helper_download_network_cached
-
-
-class NetworkCache:
-  def __init__(self, configuration_path):
-    if not os.path.exists(configuration_path):
-      raise ValueError("You need configuration file")
-    self.configuration = configuration_path
-    self._load()
- 
-  def _load(self):
-    network_cache_info = configparser.RawConfigParser()
-    network_cache_info.read(self.configuration)
-    network_cache_info_dict = dict(network_cache_info.items('networkcache'))
-    def get_(name):
-      return network_cache_info_dict.get(name)
-
-    self.download_binary_cache_url = get_('download-binary-cache-url')
-    self.download_cache_url = get_('download-cache-url')
-    self.download_binary_dir_url = get_('download-binary-dir-url')
-    self.signature_certificate_list = get_('signature-certificate-list')
-
-    # Not mandatory
-    self.dir_url = get_('upload-dir-url')
-    self.cache_url = get_('upload-cache-url')
-    key_file_key = 'signature-private-key-file'
-    self.signature_private_key_file = get_(key_file_key)
-    key_file_old_key = 'signature_private_key_file'
-    if key_file_old_key in network_cache_info_dict:
-      raise ValueError(
-        '%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
-    self.shacache_ca_file = get_('shacache-ca-file')
-    self.shacache_cert_file = get_('shacache-cert-file')
-    self.shacache_key_file = get_('shacache-key-file')
-    self.shadir_cert_file = get_('shadir-cert-file')
-    self.shadir_key_file = get_('shadir-key-file')
-    self.shadir_ca_file = get_('shadir-ca-file')
-
-    if network_cache_info.has_section('shacache'):
-      self.key = network_cache_info.get('shacache', 'key')
-    else:
-      self.key = "slapos-upgrade-testing-key"
-
-  def upload(self, path, metadata_dict, is_sha256file=False):
-    """
-    Upload an existing file, using a file_descriptor.
-    """
-    if is_sha256file: 
-      key = self.key + "-sha256-content"
-    else:
-      key = self.key
-
-    file_descriptor = open(path, 'rb')
-    if not (self.dir_url and self.cache_url):
-      raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
-
-    # backward compatibility
-    metadata_dict.setdefault('file', 'notused')
-    metadata_dict.setdefault('urlmd5', 'notused')
-
-    # convert '' into None in order to call nc nicely
-    with NetworkcacheClient(self.cache_url, self.dir_url,
-          signature_private_key_file=self.signature_private_key_file or None,
-          signature_certificate_list=self.signature_certificate_list or None,
-          shacache_ca_file=self.shacache_ca_file or None,
-          shacache_cert_file=self.shacache_cert_file or None,
-          shacache_key_file=self.shacache_key_file or None,
-          shadir_cert_file=self.shadir_cert_file or None,
-          shadir_key_file=self.shadir_key_file or None,
-          shadir_ca_file=self.shadir_ca_file or None,
-
-        ) as nc:
-      return nc.upload(file_descriptor, key, **metadata_dict)
-
-
-  def download(self, path, wanted_metadata_dict={}, 
-                 required_key_list=[], strategy=None, is_sha256file=False):
-
-    if is_sha256file:
-      key = self.key + "-sha256-content"
-    else:
-      key = self.key
-
-    result = helper_download_network_cached(
-              self.download_binary_dir_url, 
-              self.download_binary_cache_url,
-              self.signature_certificate_list,
-              key, wanted_metadata_dict, 
-              required_key_list, strategy)
-
-    if result:
-      # XXX check if nc filters signature_certificate_list!
-      # Creates a file with content to desired path.
-      file_descriptor, metadata_dict = result
-      f = open(path, 'w+b')
-      try:
-        shutil.copyfileobj(file_descriptor, f)
-        # XXX method should check MD5.
-        return metadata_dict
-      finally:
-        f.close()
-        file_descriptor.close()
-    return False
-
 
 
 def strategy(entry_list):
@@ -155,13 +51,20 @@ def strategy(entry_list):
       best_entry = entry
       timestamp = entry['timestamp']
 
-  return best_entry 
+  return best_entry
 
 class Signature:
 
   def __init__(self, config, logger=None):
     self.config = config
     self.logger = logger
+    self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
+    network_cache_info = configparser.RawConfigParser()
+    network_cache_info.read(self.config.slapos_configuration)
+    if network_cache_info.has_section('shacache'):
+      self.key = network_cache_info.get('shacache', 'key')
+    else:
+      self.key = "slapos-upgrade-testing-key"
 
   def log(self, message, *args):
     if self.logger is not None:
@@ -179,32 +82,66 @@ class Signature:
     return base
 
   def save_file_hash(self, path, destination):
-    base = self.get_file_hash(path) 
+    base = self.get_file_hash(path)
     with open(destination, "wb") as f:
       f.write(base)
 
+  def _download_once(self, path, wanted_metadata_dict={},
+                 required_key_list=[], is_sha256file=False):
+
+    if is_sha256file:
+      key = self.key + "-sha256-content"
+    else:
+      key = self.key
+
+    self.log('Downloading %s...', key)
+    result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
+    entry = None
+    result = list(result)
+    if result:
+      entry = strategy(result)
+      if not entry: # XXX: this should be the choice of 'strategy' function
+        self.log("Can't find best entry matching strategy, selecting "
+            "random one between acceptable ones.")
+        entry = result[0]
+    if not entry:
+      self.log('No entry matching key %s', key)
+    else:
+      # XXX check if nc filters signature_certificate_list!
+      # Creates a file with content to desired path.
+      f = open(path, 'w+b')
+      fd_download = self.shacache.download(entry['sha512'])
+      try:
+        shutil.copyfileobj(fd_download, f)
+        # XXX method should check MD5.
+        return entry
+      finally:
+        f.close()
+        fd_download.close()
+    return False
+
   def _download(self, path):
     """
     Download a tar of the repository from cache, and untar it.
     """
-    shacache = NetworkCache(self.config.slapos_configuration)
-
-    if shacache.signature_certificate_list is None:
-      raise ValueError("You need at least one valid signature for download")
-
-    download_metadata_dict = shacache.download(path=path, 
-           required_key_list=['timestamp'], strategy=strategy)
-
+    try:
+      download_metadata_dict = self._download_once(path=path,
+              required_key_list=['timestamp'])
+    except:
+      return False
     if download_metadata_dict:
       self.log('File downloaded in %s', path)
       current_sha256 = self.get_file_hash(path)
       with tempfile.NamedTemporaryFile() as f_256:
         sha256path = f_256.name
-        if shacache.download(path=sha256path, required_key_list=['timestamp'],
-                              strategy=strategy, is_sha256file=True):
+        try:
+          sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
           self.log('sha 256 downloaded in %s', sha256path)
-          expected_sha256 = f_256.read()
+        except:
+          sha256sum_present = False
 
+        if sha256sum_present:
+          expected_sha256 = f_256.read()
           if current_sha256 == expected_sha256:
             return True
           else:
@@ -230,29 +167,32 @@ class Signature:
     """
     Creates uploads repository to cache.
     """
-    shacache = NetworkCache(self.config.slapos_configuration)
-
     sha256path = path + ".sha256"
     self.save_file_hash(path, sha256path)
- 
+
     metadata_dict = {
       # XXX: we set date from client side. It can be potentially dangerous
       # as it can be badly configured.
       'timestamp': time.time(),
-      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) 
+      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
+      # backward compatibility
+      'file': 'notused',
+      'urlmd5': 'notused'
     }
     try:
-      sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict)
+      sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
       if sha512sum:
         self.log(
           'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
-          shacache.key, sha512sum)
-        sha512sum_path = shacache.upload(
-          path=sha256path, metadata_dict=metadata_dict, is_sha256file=True)
+          self.key, sha512sum)
+        sha512sum_path = self.shacache.upload(
+            open(sha256path, 'rb'),
+            self.key + "-sha256-content",
+            **metadata_dict)
         if sha512sum_path:
           self.log(
             'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
-            shacache.key, sha512sum_path)
+            self.key, sha512sum_path)
         else:
           self.log('Fail to upload sha256file file to cache.')
       else:
@@ -263,7 +203,7 @@ class Signature:
 
   def upload(self):
     self._upload(self.config.file)
- 
+
 # Class containing all parameters needed for configuration
 class Config:
   def __init__(self, option_dict=None):