Remove NetworkCache class in favor of standard NetworkcacheClient class

WARNING: this fixes a very old bug that download-binary-dir-url was used
in place of download-dir-url. This may break existing instance of
slapcache.

slapcache/signature.py

@@ -40,110 +40,6 @@ from random import choice
 from string import ascii_lowercase
 
 from slapos.libnetworkcache import NetworkcacheClient
-from slapos.networkcachehelper import helper_download_network_cached
-
-
-class NetworkCache:
-  def __init__(self, configuration_path):
-    if not os.path.exists(configuration_path):
-      raise ValueError("You need configuration file")
-    self.configuration = configuration_path
-    self._load()
- 
-  def _load(self):
-    network_cache_info = configparser.RawConfigParser()
-    network_cache_info.read(self.configuration)
-    network_cache_info_dict = dict(network_cache_info.items('networkcache'))
-    def get_(name):
-      return network_cache_info_dict.get(name)
-
-    self.download_binary_cache_url = get_('download-binary-cache-url')
-    self.download_cache_url = get_('download-cache-url')
-    self.download_binary_dir_url = get_('download-binary-dir-url')
-    self.signature_certificate_list = get_('signature-certificate-list')
-
-    # Not mandatory
-    self.dir_url = get_('upload-dir-url')
-    self.cache_url = get_('upload-cache-url')
-    key_file_key = 'signature-private-key-file'
-    self.signature_private_key_file = get_(key_file_key)
-    key_file_old_key = 'signature_private_key_file'
-    if key_file_old_key in network_cache_info_dict:
-      raise ValueError(
-        '%s is not supported anymore, use %s' % (key_file_old_key, key_file_key))
-    self.shacache_ca_file = get_('shacache-ca-file')
-    self.shacache_cert_file = get_('shacache-cert-file')
-    self.shacache_key_file = get_('shacache-key-file')
-    self.shadir_cert_file = get_('shadir-cert-file')
-    self.shadir_key_file = get_('shadir-key-file')
-    self.shadir_ca_file = get_('shadir-ca-file')
-
-    if network_cache_info.has_section('shacache'):
-      self.key = network_cache_info.get('shacache', 'key')
-    else:
-      self.key = "slapos-upgrade-testing-key"
-
-  def upload(self, path, metadata_dict, is_sha256file=False):
-    """
-    Upload an existing file, using a file_descriptor.
-    """
-    if is_sha256file: 
-      key = self.key + "-sha256-content"
-    else:
-      key = self.key
-
-    file_descriptor = open(path, 'rb')
-    if not (self.dir_url and self.cache_url):
-      raise ValueError("upload-dir-url and/or upload-cache-url is not defined")
-
-    # backward compatibility
-    metadata_dict.setdefault('file', 'notused')
-    metadata_dict.setdefault('urlmd5', 'notused')
-
-    # convert '' into None in order to call nc nicely
-    with NetworkcacheClient(self.cache_url, self.dir_url,
-          signature_private_key_file=self.signature_private_key_file or None,
-          signature_certificate_list=self.signature_certificate_list or None,
-          shacache_ca_file=self.shacache_ca_file or None,
-          shacache_cert_file=self.shacache_cert_file or None,
-          shacache_key_file=self.shacache_key_file or None,
-          shadir_cert_file=self.shadir_cert_file or None,
-          shadir_key_file=self.shadir_key_file or None,
-          shadir_ca_file=self.shadir_ca_file or None,
-
-        ) as nc:
-      return nc.upload(file_descriptor, key, **metadata_dict)
-
-
-  def download(self, path, wanted_metadata_dict={}, 
-                 required_key_list=[], strategy=None, is_sha256file=False):
-
-    if is_sha256file:
-      key = self.key + "-sha256-content"
-    else:
-      key = self.key
-
-    result = helper_download_network_cached(
-              self.download_binary_dir_url, 
-              self.download_binary_cache_url,
-              self.signature_certificate_list,
-              key, wanted_metadata_dict, 
-              required_key_list, strategy)
-
-    if result:
-      # XXX check if nc filters signature_certificate_list!
-      # Creates a file with content to desired path.
-      file_descriptor, metadata_dict = result
-      f = open(path, 'w+b')
-      try:
-        shutil.copyfileobj(file_descriptor, f)
-        # XXX method should check MD5.
-        return metadata_dict
-      finally:
-        f.close()
-        file_descriptor.close()
-    return False
-
 
 
 def strategy(entry_list):
@@ -155,13 +51,20 @@ def strategy(entry_list):
       best_entry = entry
       timestamp = entry['timestamp']
 
-  return best_entry 
+  return best_entry
 
 class Signature:
 
   def __init__(self, config, logger=None):
     self.config = config
     self.logger = logger
+    self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r'))
+    network_cache_info = configparser.RawConfigParser()
+    network_cache_info.read(self.config.slapos_configuration)
+    if network_cache_info.has_section('shacache'):
+      self.key = network_cache_info.get('shacache', 'key')
+    else:
+      self.key = "slapos-upgrade-testing-key"
 
   def log(self, message, *args):
     if self.logger is not None:
@@ -179,32 +82,66 @@ class Signature:
     return base
 
   def save_file_hash(self, path, destination):
-    base = self.get_file_hash(path) 
+    base = self.get_file_hash(path)
     with open(destination, "wb") as f:
       f.write(base)
 
+  def _download_once(self, path, wanted_metadata_dict={},
+                 required_key_list=[], is_sha256file=False):
+
+    if is_sha256file:
+      key = self.key + "-sha256-content"
+    else:
+      key = self.key
+
+    self.log('Downloading %s...', key)
+    result = self.shacache.select(key, wanted_metadata_dict, required_key_list)
+    entry = None
+    result = list(result)
+    if result:
+      entry = strategy(result)
+      if not entry: # XXX: this should be the choice of 'strategy' function
+        self.log("Can't find best entry matching strategy, selecting "
+            "random one between acceptable ones.")
+        entry = result[0]
+    if not entry:
+      self.log('No entry matching key %s', key)
+    else:
+      # XXX check if nc filters signature_certificate_list!
+      # Creates a file with content to desired path.
+      f = open(path, 'w+b')
+      fd_download = self.shacache.download(entry['sha512'])
+      try:
+        shutil.copyfileobj(fd_download, f)
+        # XXX method should check MD5.
+        return entry
+      finally:
+        f.close()
+        fd_download.close()
+    return False
+
   def _download(self, path):
     """
     Download a tar of the repository from cache, and untar it.
     """
-    shacache = NetworkCache(self.config.slapos_configuration)
-
-    if shacache.signature_certificate_list is None:
-      raise ValueError("You need at least one valid signature for download")
-
-    download_metadata_dict = shacache.download(path=path, 
-           required_key_list=['timestamp'], strategy=strategy)
-
+    try:
+      download_metadata_dict = self._download_once(path=path,
+              required_key_list=['timestamp'])
+    except:
+      return False
     if download_metadata_dict:
       self.log('File downloaded in %s', path)
       current_sha256 = self.get_file_hash(path)
       with tempfile.NamedTemporaryFile() as f_256:
         sha256path = f_256.name
-        if shacache.download(path=sha256path, required_key_list=['timestamp'],
-                              strategy=strategy, is_sha256file=True):
+        try:
+          sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True)
           self.log('sha 256 downloaded in %s', sha256path)
-          expected_sha256 = f_256.read()
+        except:
+          sha256sum_present = False
 
+        if sha256sum_present:
+          expected_sha256 = f_256.read()
           if current_sha256 == expected_sha256:
             return True
           else:
@@ -230,29 +167,32 @@ class Signature:
     """
     Creates uploads repository to cache.
     """
-    shacache = NetworkCache(self.config.slapos_configuration)
-
     sha256path = path + ".sha256"
     self.save_file_hash(path, sha256path)
- 
+
     metadata_dict = {
       # XXX: we set date from client side. It can be potentially dangerous
       # as it can be badly configured.
       'timestamp': time.time(),
-      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) 
+      'token': ''.join([choice(ascii_lowercase) for _ in range(128)]),
+      # backward compatibility
+      'file': 'notused',
+      'urlmd5': 'notused'
     }
     try:
-      sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict)
+      sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict)
       if sha512sum:
         self.log(
           'Uploaded %s to cache (using %s key) with SHA512 %s.', path,
-          shacache.key, sha512sum)
-        sha512sum_path = shacache.upload(
-          path=sha256path, metadata_dict=metadata_dict, is_sha256file=True)
+          self.key, sha512sum)
+        sha512sum_path = self.shacache.upload(
+            open(sha256path, 'rb'),
+            self.key + "-sha256-content",
+            **metadata_dict)
         if sha512sum_path:
           self.log(
             'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path,
-            shacache.key, sha512sum_path)
+            self.key, sha512sum_path)
         else:
           self.log('Fail to upload sha256file file to cache.')
       else:
@@ -263,7 +203,7 @@ class Signature:
 
   def upload(self):
     self._upload(self.config.file)
- 
+
 # Class containing all parameters needed for configuration
 class Config:
   def __init__(self, option_dict=None):

slapcache/template/update.cfg.in

@@ -5,6 +5,7 @@ key = %(key)s
 download-binary-cache-url = http://shacache.nxdcdn.com
 download-cache-url = https://shacache.nxdcdn.com
 download-binary-dir-url = http://shadir.nxdcdn.com
+download-dir-url = http://shadir.nxdcdn.com
 
 # Standard Signature handled by SlapOS Administrators
 signature-certificate-list =