download: add support for slapos.libnetworkcache

This is a rewrite of commit 6e4b8efe
("Support network cache in Download.download()").

src/zc/buildout/buildout.py

@@ -245,7 +245,7 @@ _buildout_default_options = _annotate_section({
     'use-dependency-links': 'true',
     }, 'DEFAULT_VALUE')
 
-network_cache_parameter_dict = {}
+networkcache_client = None
 
 class Buildout(DictMixin):
 
@@ -516,27 +516,16 @@ class Buildout(DictMixin):
         networkcache_section_name = options.get('networkcache-section')
         if networkcache_section_name:
             networkcache_section = self[networkcache_section_name]
-            for k in (
-                'download-cache-url',
-                'download-dir-url',
-                'upload-cache-url',
-                'upload-dir-url',
-                'signature-certificate-list',
-                'signature-private-key-file',
-                'shacache-ca-file',
-                'shacache-cert-file',
-                'shacache-key-file',
-                'shadir-ca-file',
-                'shadir-cert-file',
-                'shadir-key-file',
-            ):
-                network_cache_parameter_dict[k] = networkcache_section.get(k, '')
-                # parse signature list
-                cert_marker = '-----BEGIN CERTIFICATE-----'
-            network_cache_parameter_dict['signature-certificate-list'] = \
-                [cert_marker + '\n' + q.strip() \
-                    for q in network_cache_parameter_dict['signature-certificate-list'].split(cert_marker) \
-                    if q.strip()]
+            try:
+                from slapos.libnetworkcache import NetworkcacheClient
+                global networkcache_client
+                networkcache_client = NetworkcacheClient(networkcache_section)
+            except ImportError:
+                pass
+            except Exception:
+                self._logger.exception("There was problem while trying to"
+                    " import slapos.libnetworkcache."
+                    " Networkcache forced to be disabled.")
 
     def _buildout_path(self, name):
         if '${' in name:

src/zc/buildout/download.py

@@ -20,13 +20,14 @@ except ImportError:
 
 try:
     # Python 3
+    from urllib.error import HTTPError
     from urllib.request import Request, splitport, splituser, urlopen
     from urllib.parse import urlparse, urlunparse
 except ImportError:
     # Python 2
     from urlparse import urlparse
     from urlparse import urlunparse
-    from urllib2 import Request, splitport, splituser, urlopen
+    from urllib2 import HTTPError, Request, splitport, splituser, urlopen
 
 from zc.buildout.easy_install import realpath
 from base64 import b64encode
@@ -216,47 +217,32 @@ class Download(object):
             if not path:
                 handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
                 os.close(handle)
-            from .buildout import network_cache_parameter_dict as nc
-            if not download_network_cached(
-                nc.get('download-dir-url'),
-                nc.get('download-cache-url'),
-                tmp_path, url, self.logger,
-                nc.get('signature-certificate-list'), md5sum):
-                # Download from original url if not cached or md5sum doesn't match.
-                try:
-                    tmp_path, headers = self.urlretrieve(url, tmp_path)
-                except HTTPError:
-                    if not alternate_url:
-                        raise
-                    self.logger.info('using alternate URL: %s', alternate_url)
-                    download_url = alternate_url
-                    tmp_path, headers = self.urlretrieve(
-                        alternate_url, tmp_path)
-                if not check_md5sum(tmp_path, md5sum):
-                    raise ChecksumError(
-                        'MD5 checksum mismatch downloading %r' % download_url)
-                # Upload the file to network cache.
-                if nc.get('upload-cache-url') and nc.get('upload-dir-url'):
-                    upload_network_cached(
-                        nc.get('upload-dir-url'),
-                        nc.get('upload-cache-url'), url, tmp_path, self.logger,
-                        nc.get('signature-private-key-file'),
-                        nc.get('shacache-ca-file'),
-                        nc.get('shacache-cert-file'),
-                        nc.get('shacache-key-file'),
-                        nc.get('shadir-ca-file'),
-                        nc.get('shadir-cert-file'),
-                        nc.get('shadir-key-file'))
+            self._download(url, tmp_path, md5sum, alternate_url)
             cleanup = False
-        except IOError as e:
-            raise zc.buildout.UserError("Error downloading %s: %s"
-                                        % (download_url, e))
         finally:
             if cleanup and tmp_path:
                 remove(tmp_path)
 
         return tmp_path, not path
 
+    def _download(self, url, path, md5sum=None, alternate_url=None):
+        download_url = url
+        try:
+            try:
+                self.urlretrieve(url, path)
+            except HTTPError:
+                if not alternate_url:
+                    raise
+                self.logger.info('using alternate URL: %s', alternate_url)
+                download_url = alternate_url
+                self.urlretrieve(alternate_url, path)
+            if not check_md5sum(path, md5sum):
+                raise ChecksumError('MD5 checksum mismatch downloading %r'
+                                    % download_url)
+        except IOError as e:
+            raise zc.buildout.UserError("Error downloading %s: %s"
+                                        % (download_url, e))
+
     def filename(self, url):
         """Determine a file name from a URL according to the configuration.
 
@@ -306,6 +292,32 @@ class Download(object):
             return tmp_path, src.info()
 
 
+class Download(Download):
+
+    def _download(self, url, path, md5sum=None, alternate_url=None):
+        from .buildout import networkcache_client as nc
+        key = 'slapos-buildout-' + md5(url.encode()).hexdigest()
+        if nc and nc.tryDownload(key):
+            with nc:
+                # BBB: To avoid collision, wanted_metadata_dict should be
+                #      {'url': url}
+                entry = next(nc.select(key), None)
+                if entry is None:
+                    err = 'no matching entry'
+                else:
+                    with closing(nc.download(entry['sha512'])) as src, \
+                         open(path, 'wb') as dst:
+                        shutil.copyfileobj(src, dst)
+                    if check_md5sum(path, md5sum):
+                        return
+                    err = 'MD5 checksum mismatch'
+                self.logger.info('Cannot download from network cache: %s', err)
+        super(Download, self)._download(url, path, md5sum, alternate_url)
+        if nc and nc.tryUpload(key):
+            with nc, open(path, 'rb') as f:
+                nc.upload(f, key, url=url)
+
+
 def check_md5sum(path, md5sum):
     """Tell whether the MD5 checksum of the file at path matches.
 
@@ -331,8 +343,6 @@ def remove(path):
     if os.path.exists(path):
         os.remove(path)
 
-from zc.buildout.networkcache import \
-  download_network_cached, upload_network_cached, HTTPError
 
 def locate_at(source, dest):
     if dest is None or realpath(dest) == realpath(source):