gitclone.py 9.28 KB
Newer Older
Antoine Catton's avatar
Antoine Catton committed
1 2
##############################################################################
#
3
# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
Antoine Catton's avatar
Antoine Catton committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

28
import hashlib
Antoine Catton's avatar
Antoine Catton committed
29
import os
30 31 32 33 34 35 36 37
import shutil
import time
import traceback

from zc.buildout import UserError
from subprocess import check_call, CalledProcessError

try:
38 39 40 41 42
  try:
    from slapos.networkcachehelper import \
       helper_upload_network_cached_from_directory, \
       helper_download_network_cached_to_directory
  except ImportError:
43
    LIBNETWORKCACHE_ENABLED = False
44 45 46 47 48 49 50
  else:
    LIBNETWORKCACHE_ENABLED = True
except:
  print 'There was problem while trying to import slapos.libnetworkcache:'\
      '\n%s' % traceback.format_exc()
  LIBNETWORKCACHE_ENABLED = False
  print 'Networkcache forced to be disabled.'
Antoine Catton's avatar
Antoine Catton committed
51 52 53

GIT_DEFAULT_REMOTE_NAME = 'origin'
GIT_DEFAULT_BRANCH_NAME = 'master'
54
TRUE_VALUES = ('y', 'yes', '1', 'true')
Antoine Catton's avatar
Antoine Catton committed
55

56 57 58 59
GIT_CLONE_ERROR_MESSAGE = 'Impossible to clone repository.'
GIT_CLONE_CACHE_ERROR_MESSAGE = 'Impossible to clone repository and ' \
    'impossible to download from cache.'

60 61 62 63
def upload_network_cached(path, name, revision, networkcache_options):
  """
  Creates uploads repository to cache.
  """
64
  if not (LIBNETWORKCACHE_ENABLED and networkcache_options.get(
65
      'upload-dir-url')):
66
    return False
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  try:
    print 'Uploading git repository to cache...'
    metadata_dict = {
        'revision':revision,
        # XXX: we set date from client side. It can be potentially dangerous
        # as it can be badly configured.
        'timestamp':time.time(),
    }
    helper_upload_network_cached_from_directory(
        path=path,
        directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
        metadata_dict=metadata_dict,
        # Then we give a lot of not interesting things
        dir_url=networkcache_options.get('upload-dir-url'),
        cache_url=networkcache_options.get('upload-cache-url'),
        signature_private_key_file=networkcache_options.get(
            'signature-private-key-file'),
        shacache_cert_file=networkcache_options.get('shacache-cert-file'),
        shacache_key_file=networkcache_options.get('shacache-key-file'),
        shadir_cert_file=networkcache_options.get('shadir-cert-file'),
        shadir_key_file=networkcache_options.get('shadir-key-file'),
    )
    print 'Uploaded git repository to cache.'
  except Exception:
    print 'Unable to upload to cache:\n%s.' % traceback.format_exc()


def download_network_cached(path, name, revision, networkcache_options):
  """
  Download a tar of the repository from cache, and untar it.
  """
  def strategy(entry_list):
    """
    Get the latest entry.
    """
    timestamp = 0
    best_entry = None
    for entry in entry_list:
      if entry['timestamp'] > timestamp:
        best_entry = entry
    return best_entry

  return helper_download_network_cached_to_directory(
      path=path,
      directory_key='git-buildout-%s' % hashlib.md5(name).hexdigest(),
      wanted_metadata_dict={'revision':revision},
      required_key_list=['timestamp'],
      strategy=strategy,
      # Then we give a lot of not interesting things
      dir_url=networkcache_options.get('download-dir-url'),
      cache_url=networkcache_options.get('download-cache-url'),
      signature_certificate_list=\
          networkcache_options.get('signature-certificate-list'),
  )

Antoine Catton's avatar
Antoine Catton committed
122
class Recipe(object):
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
  """Clone a git repository.

  Input:

    repository
      Address of the remote repository.

    git-executable
      Path to the git executable to use.
    
    revision (optional)
      Revision to use.
      
    branch (optional)
      Branch to use.

    location (optional)
      Location 

    - In order to prevent buildout uninstalling, and re-installing parts
      at each time revision input changes, you can use an additional
      section to specifiy a revision. By inserting a new section in
      your buildout configuration:
      
        [<name>-override]
        revision = <xxx>

      Where <name> is the name the current section, and <xxx>
      the revision to use.
    - You cannot use the revision input and the additional
      section descripted above at the same time.
    - You can't specify a branch and a revision at the same time.

  """
Antoine Catton's avatar
Antoine Catton committed
157

158
  def __init__(self, buildout, name, options):
159 160
    options.setdefault('location',
        os.path.join(buildout['buildout']['parts-directory'], name))
161
    self.repository = options.get('repository')
162
    self.branch = options.get('branch', GIT_DEFAULT_BRANCH_NAME)
163
    # Get revision
164
    self.revision = options.get('revision')
165 166 167 168 169
    # Try to get revision in [<name>-override] section.
    if buildout.get('%s-override' %name):
      if buildout.get('%s-override' %name).get('revision'):
        self.revision = buildout.get('%s-override' %name).get('revision')
      
170 171
    self.git_command = options.get('git-executable', 'git')
    self.name = name
172
    self.location = options.get('location')
173
    # Set boolean values
174 175
    for key in ('develop', 'use-cache'):
      setattr(self, key.replace('-', '_'), options.get(key) in TRUE_VALUES)
176

177
    self.networkcache = buildout.get('networkcache', {})
178

179 180 181
    # Check if input is correct
    if not self.repository:
      raise UserError('repository parameter is mandatory.')
182
    if self.revision and self.branch != GIT_DEFAULT_BRANCH_NAME:
183
      # revision and branch options are incompatible
184 185
      raise UserError('revision and branch (other than master) parameters '
          'are set but are incompatible. Please specify only one of them.')
Antoine Catton's avatar
Antoine Catton committed
186

187 188
  def gitReset(self, revision=None):
    """Operates git reset on the repository."""
189
    command = [self.git_command, 'reset', '--hard']
190 191
    if revision:
      command.append(revision)
192 193
    check_call(command, cwd=self.location)

Antoine Catton's avatar
Antoine Catton committed
194

195
  def install(self):
196 197 198 199
    """
    Do a git clone.
    If branch is specified, checkout to it.
    If revision is specified, reset to it.
200 201
    If something fails, try to download from cache.
    Else, if possible, try to upload to cache.
202
    """
203 204 205 206 207 208 209 210 211 212 213 214
    # If directory already exist: delete it.
    if os.path.exists(self.location):
      print 'destination directory already exists. Deleting it.'
      shutil.rmtree(self.location)

    git_clone_command = [self.git_command, 'clone',
                self.repository,
                self.location]
    if self.branch:
      git_clone_command.extend(['--branch', self.branch])

    try:
215
      check_call(git_clone_command)
216 217 218 219
      if not os.path.exists(self.location):
        raise UserError("Unknown error while cloning repository.")
      if self.revision:
        self.gitReset(self.revision)
220 221 222
      if self.use_cache:
        upload_network_cached(os.path.join(self.location, '.git'),
                              self.repository, self.revision, self.networkcache)
223 224 225
    except CalledProcessError:
      print ("Unable to download from git repository. Trying from network "
          "cache...")
226 227
      if os.path.exists(self.location):
        shutil.rmtree(self.location)
228
      if not self.use_cache:
229
        raise UserError(GIT_CLONE_ERROR_MESSAGE)
230 231 232
      os.mkdir(self.location)
      if not download_network_cached(os.path.join(self.location, '.git'),
          self.repository, self.revision, self.networkcache):
233
        raise UserError(GIT_CLONE_CACHE_ERROR_MESSAGE)
234
      self.gitReset()
Antoine Catton's avatar
Antoine Catton committed
235

236
    return [self.location]
Antoine Catton's avatar
Antoine Catton committed
237

238 239 240 241 242 243 244 245 246 247 248
  def deletePycFiles(self, path):
    """Delete *.pyc files so that deleted/moved files can not be imported"""
    for path, dir_list, file_list in os.walk(path):
      for file in file_list:
        if file[-4:] in ('.pyc', '.pyo'):
          # allow several processes clean the same folder at the same time
          try:
            os.remove(os.path.join(path, file))
          except OSError, e:
            if e.errno != errno.ENOENT:
              raise
Antoine Catton's avatar
Antoine Catton committed
249

250
  def update(self):
251 252
    """
    Do a git fetch.
253
    If user doesn't develop, reset to remote revision (or branch if revision is
254
    not specified).
255
    """
256 257 258 259
    # first cleanup pyc files
    self.deletePycFiles(self.location)

    # then update
260
    check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
Antoine Catton's avatar
Antoine Catton committed
261

262 263
    # If develop parameter is set, don't reset/update.
    # Otherwise, reset --hard
264 265 266
    if not self.develop:
      if self.revision:
        self.gitReset(self.revision)
267
      else:
268
        self.gitReset('@{upstream}')