Add support for networkcache

setup.py

@@ -24,6 +24,7 @@ setup(name=name,
       install_requires=[
         'setuptools', # namespaces
         'zc.buildout', # plays with buildout
+        'slapos.libnetworkcache>=0.13.1', # Uses helper new in this version
         ],
       zip_safe=True,
       entry_points={

slapos/recipe/gitclone.py

@@ -25,35 +25,117 @@
 #
 ##############################################################################
 
+import hashlib
 import os
-
-from subprocess import check_call
+import shutil
+import time
+import traceback
+
+from zc.buildout import UserError
+from subprocess import check_call, CalledProcessError
+
+try:
+    try:
+        from slapos.networkcachehelper import \
+           helper_upload_network_cached_from_directory, \
+           helper_download_network_cached_to_directory
+    except ImportError:
+        LIBNETWORKCACHE_ENABLED = False
+    else:
+        LIBNETWORKCACHE_ENABLED = True
+except:
+    print 'There was problem while trying to import slapos.libnetworkcache:'\
+        '\n%s' % traceback.format_exc()
+    LIBNETWORKCACHE_ENABLED = False
+    print 'Networkcache forced to be disabled.'
 
 GIT_DEFAULT_REMOTE_NAME = 'origin'
 GIT_DEFAULT_BRANCH_NAME = 'master'
 TRUE_VALUES = ['y', 'yes', '1', 'true']
 
+def upload_network_cached(path, name, revision, networkcache_options):
+  """
+  Creates uploads repository to cache.
+  """
+  try:
+    print 'Uploading git repository to cache...'
+    metadata_dict = {
+        'revision':revision,
+        # XXX: we set date from client side. It can be potentially dangerous
+        # as it can be badly configured.
+        'timestamp':time.time(),
+    }
+    helper_upload_network_cached_from_directory(
+        path=path,
+        directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
+        metadata_dict=metadata_dict,
+        # Then we give a lot of not interesting things
+        dir_url=networkcache_options.get('upload-dir-url'),
+        cache_url=networkcache_options.get('upload-cache-url'),
+        signature_private_key_file=networkcache_options.get(
+            'signature-private-key-file'),
+        shacache_cert_file=networkcache_options.get('shacache-cert-file'),
+        shacache_key_file=networkcache_options.get('shacache-key-file'),
+        shadir_cert_file=networkcache_options.get('shadir-cert-file'),
+        shadir_key_file=networkcache_options.get('shadir-key-file'),
+    )
+    print 'Uploaded git repository to cache.'
+  except Exception:
+    print 'Unable to upload to cache:\n%s.' % traceback.format_exc()
+
+
+def download_network_cached(path, name, revision, networkcache_options):
+  """
+  Download a tar of the repository from cache, and untar it.
+  """
+  def strategy(entry_list):
+    """
+    Get the latest entry.
+    """
+    timestamp = 0
+    best_entry = None
+    for entry in entry_list:
+      if entry['timestamp'] > timestamp:
+        best_entry = entry
+    return best_entry
+
+  return helper_download_network_cached_to_directory(
+      path=path,
+      directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
+      wanted_metadata_dict={'revision':revision},
+      required_key_list=['timestamp'],
+      strategy=strategy,
+      # Then we give a lot of not interesting things
+      dir_url=networkcache_options.get('download-dir-url'),
+      cache_url=networkcache_options.get('download-cache-url'),
+      signature_certificate_list=\
+          networkcache_options.get('signature-certificate-list'),
+  )
+
 class Recipe(object):
   """Clone a git repository."""
 
   def __init__(self, buildout, name, options):
-    self.url = options.get('url')
+    self.repository = options.get('repository')
     self.branch = options.get('branch', GIT_DEFAULT_BRANCH_NAME)
     self.revision = options.get('revision')
-    self.git_command = options.get('git-command', 'git')
+    self.git_command = options.get('git-executable', 'git')
+    self.name = name
     self.location = options.get('location',
         os.path.join(buildout['buildout']['parts-directory'], name))
     if options.get('develop') in TRUE_VALUES:
       self.develop = True
 
-    if not self.url:
-      raise ValueError('url parameter is mandatory.')
+    # XXX clean
+    self.networkcache = buildout.get('networkcache')
 
+    # Check if input is correct
+    if not self.repository:
+      raise UserError('repository parameter is mandatory.')
     if self.revision and self.branch != GIT_DEFAULT_BRANCH_NAME:
       # revision and branch options are incompatible
-      raise ValueError('revision and branch parameters are set but are '
-          'incompatible. Please specify only one of them.')
-
+      raise UserError('revision and branch (other than master) parameters '
+          'are set but are incompatible. Please specify only one of them.')
 
   def gitReset(self, revision=None):
     """Operates git reset on the repository."""
@@ -68,17 +150,37 @@ class Recipe(object):
     Do a git clone.
     If branch is specified, checkout to it.
     If revision is specified, reset to it.
+    If something fails, try to download from cache.
+    Else, if possible, try to upload to cache.
     """
-    if not os.path.exists(self.location):
-      git_clone_command = [self.git_command, 'clone',
-                  self.url,
-                  self.location]
-      if self.branch:
-        git_clone_command.extend(['--branch', self.branch])
+    # If directory already exist: delete it.
+    if os.path.exists(self.location):
+      print 'destination directory already exists. Deleting it.'
+      shutil.rmtree(self.location)
+
+    git_clone_command = [self.git_command, 'clone',
+                self.repository,
+                self.location]
+    if self.branch:
+      git_clone_command.extend(['--branch', self.branch])
+
+    try:
       check_call(git_clone_command)
-
-    if self.revision:
-      self.gitReset(self.revision)
+      if not os.path.exists(self.location):
+        raise UserError("Unknown error while cloning repository.")
+      if self.revision:
+        self.gitReset(self.revision)
+      upload_network_cached(self.location, self.name, self.revision,
+          self.networkcache)
+    except CalledProcessError:
+      print ("Unable to download from git repository. Trying from network "
+          "cache...")
+      if not download_network_cached(self.location, self.name, self.revision, 
+          self.networkcache):
+        if os.path.exists(self.location):
+          shutil.rmtree(self.location)
+        raise UserError('Impossible to clone repository and impossible to '
+            'download from cache.')
 
     return [self.location]
 
@@ -87,7 +189,7 @@ class Recipe(object):
     """
     Do a git fetch.
     If user doesn't develop, reset to remote revision (or branch if revision is
-    not specified.)
+    not specified).
     """
     check_call([self.git_command, 'fetch', '--all'], cwd=self.location)