Now slapos.buildout is working again if the new implementation.

After this commit it is possible to download and upload anything to
SHACACHE and SHADIR servers.

It uses slapos.libnetworkcache to handle the download and upload.

src/zc/buildout/networkcache.py

@@ -17,8 +17,10 @@ import hashlib
 import os
 import posixpath
 import re
+import urllib
+import urlparse
 from download import check_md5sum
-from slapos.tool.libnetworkcache import NetworkcacheClient
+from slapos.libnetworkcache import NetworkcacheClient, UploadError
 
 
 _md5_re = re.compile(r'md5=([a-f0-9]+)')
@@ -34,7 +36,7 @@ def _get_md5_from_url(url):
 def _update_network_cached_log(path, url):
     """Update the networkcached.log file.
 
-    Adding the last key and URL which has been retrieved from networkcached server.
+    Adding the last key and URL which has been retrieved from shacache server.
 
     """
     f = open(path, 'r')
@@ -63,7 +65,31 @@ def _update_network_cached_log(path, url):
         f.close()
 
 
-def download_network_cached(network_cache, path, url, logger, md5sum=None):
+def select_sha512sum_from_shadir(sha_dir, url, logger):
+    """It must return the sha512sum if exists on selected directory. """
+    # On buildout.cfg the option sha-dir is the complete link including the
+    # directory key, so we must to extract it to use the libnetworkcache
+    # correctly.
+    directory_key = sha_dir.split('/')[-1]
+    sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
+
+    try:
+      nc = NetworkcacheClient(shadir=sha_dir_without_directory_key)
+      data_list = nc.select(directory_key)
+    except (IOError, UploadError), e:
+      logger.info('Could not retrieve the information from SHADIR. %s' % e)
+      return None
+
+    urlmd5 = hashlib.md5(url).hexdigest()
+    for data in data_list:
+      information_dict, signature = data
+      if information_dict.get('urlmd5') == urlmd5:
+        return information_dict.get('sha512')
+
+    return None
+
+
+def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
     """Download from a network cache provider
 
     If something fail (providor be offline, or hash_string fail), we ignore
@@ -71,68 +97,63 @@ def download_network_cached(network_cache, path, url, logger, md5sum=None):
 
     return True if download succeeded.
     """
-    if network_cache in (None, '',):
+    if sha_cache in (None, '',):
         # Not able to use network cache
         return False
     if md5sum is None:
-        print "URL ", url
-        md5sum = _get_md5_from_url(url) 
-
-    # XXX (lucas): This will be removed.
-    key = hashlib.sha512(url).hexdigest()
+        md5sum = _get_md5_from_url(url)
 
-    network_cached_url = os.path.join(network_cache, key)
-    logger.info('Downloading from network cache %s' % network_cached_url)
-    try:
-        nc = NetworkcacheClient(network_cache)
-        result = nc.get(key)
 
-        if result.status != 200:
-          logger.info('Fail to download from network cache: File Not Found.')
-          return False
-
-        file_content = result.read()
-        # save the file
-        f = open(path, 'w+b')
-        try:
-          f.write(file_content)
-        finally:
-          f.close()
+    sha512sum = select_sha512sum_from_shadir(sha_dir, url, logger)
+    if sha512sum is None:
+      return False
 
+    sha_cache_url = os.path.join(sha_cache, sha512sum)
+    logger.info('Downloading from network cache %s' % sha_cache_url)
+    try:
+        nc = NetworkcacheClient(shacache=sha_cache)
+        nc.download(path, sha512sum)
         if not check_md5sum(path, md5sum):
-            logger.info('MD5 checksum mismatch downloading %r' % \
-                                                           network_cached_url)
+            logger.info('MD5 checksum mismatch downloading %r' % sha_cache_url)
             return False
     except IOError, e:
         logger.info('Fail to download from network cache %s: %s' % \
-                                                  (network_cached_url, str(e)))
+                                                  (sha_cache_url, str(e)))
         return False
-
     return True
 
 
-def upload_network_cached(network_cache, external_url, path, logger):
-    """Upload file to a network cache server"""
-    if network_cache in [None, '']:
+def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
+    """Upload file to a shacache server"""
+    if sha_cache in [None, ''] or sha_dir in [None, '']:
         logger.debug(
-         'Upload cache ignored, network-cache was not provided')
+             'Upload cache ignored, shacache or shadir was not provided!')
         return False
 
+    # On buildout.cfg the option sha-dir is the complete link including the
+    # directory key, so we must to extract it to use the libnetworkcache
+    # correctly.
+    directory_key = sha_dir.split('/')[-1]
+    sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
+
+    f = open(path, 'r')
     try:
-        f = open(path, 'r')
-        try:
-          file_content = f.read()
-        finally:
-          f.close()
-
-        nc = NetworkcacheClient(network_cache)
-        result = nc.put(file_content)
-        if result.status == 200:
-            _update_network_cached_log(path, external_url)
-
-    except (IOError, EOFError), e:
-        logger.info('Fail to upload cache on %s. %s' % \
-                                                  (external_url, str(e)))
+
+        kw = dict(file_name=os.path.basename(external_url),
+                  urlmd5=hashlib.md5(external_url).hexdigest())
+
+        nc = NetworkcacheClient(shacache=sha_cache,
+                                shadir=sha_dir_without_directory_key)
+        data = nc.upload(f, directory_key, **kw)
+        _update_network_cached_log(path, external_url)
+    except (IOError, UploadError), e:
+        logger.info('Fail to upload file. %s' % \
+                                                  (str(e)))
+        return False
+
+    finally:
+      f.close()
+
     return True