Updater.py 7.78 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
27 28 29
import errno
import os
import re
30
import shutil
31 32 33 34
import subprocess
import sys
import threading

35 36
from testnode import SubprocessError

37 38 39
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)

40 41 42 43

GIT_TYPE = 'git'
SVN_TYPE = 'svn'

44
class Updater(object):
45 46 47 48

  _git_cache = {}
  stdin = file(os.devnull)

49
  def __init__(self, repository_path, log, revision=None, git_binary=None,
50 51
      branch=None, realtime_output=True, process_manager=None, url=None,
      working_directory=None):
52
    self.log = log
53 54
    self.revision = revision
    self._path_list = []
55
    self.branch = branch
56 57
    self.repository_path = repository_path
    self.git_binary = git_binary
58
    self.realtime_output = realtime_output
59
    self.process_manager = process_manager
60 61
    self.url = url
    self.working_directory = working_directory
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94

  def getRepositoryPath(self):
    return self.repository_path

  def getRepositoryType(self):
    try:
      return self.repository_type
    except AttributeError:
      # guess the type of repository we have
      if os.path.isdir(os.path.join(
                       self.getRepositoryPath(), '.git')):
        repository_type = GIT_TYPE
      elif os.path.isdir(os.path.join(
                       self.getRepositoryPath(), '.svn')):
        repository_type = SVN_TYPE
      else:
        raise NotImplementedError
      self.repository_type = repository_type
      return repository_type

  def deletePycFiles(self, path):
    """Delete *.pyc files so that deleted/moved files can not be imported"""
    for path, dir_list, file_list in os.walk(path):
      for file in file_list:
        if file[-4:] in ('.pyc', '.pyo'):
          # allow several processes clean the same folder at the same time
          try:
            os.remove(os.path.join(path, file))
          except OSError, e:
            if e.errno != errno.ENOENT:
              raise

  def spawn(self, *args, **kw):
95 96 97
    cwd = kw.pop("cwd", None)
    if cwd is None:
      cwd = self.getRepositoryPath()
98 99
    return self.process_manager.spawn(*args, 
                                      log_prefix='git',
100
                                      cwd=cwd,
101
                                      **kw)
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129

  def _git(self, *args, **kw):
    return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()

  def _git_find_rev(self, ref):
    try:
      return self._git_cache[ref]
    except KeyError:
      if os.path.exists('.git/svn'):
        r = self._git('svn', 'find-rev', ref)
        assert r
        self._git_cache[ref[0] != 'r' and 'r%u' % int(r) or r] = ref
      else:
        r = self._git('rev-list', '--topo-order', '--count', ref), ref
      self._git_cache[ref] = r
      return r

  def getRevision(self, *path_list):
    if not path_list:
      path_list = self._path_list
    if self.getRepositoryType() == GIT_TYPE:
      h = self._git('log', '-1', '--format=%H', '--', *path_list)
      return self._git_find_rev(h)
    elif self.getRepositoryType() == SVN_TYPE:
      stdout = self.spawn('svn', 'info', *path_list)['stdout']
      return str(max(map(int, SVN_CHANGED_REV.findall(stdout))))
    raise NotImplementedError

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
  def deleteRepository(self):
    self.log("Wrong repository or wrong url, deleting repos %s" % \
             self.repository_path)
    shutil.rmtree(self.repository_path)

  def checkRepository(self):
    # make sure that the repository is like we expect
    if self.url:
      if os.path.exists(self.repository_path):
        correct_url = False
        try:
          remote_url = self._git("config", "--get", "remote.origin.url")
          if remote_url == self.url:
            correct_url = True
        except (SubprocessError,) as e:
          self.log("SubprocessError", exc_info=sys.exc_info())
        if not(correct_url):
          self.deleteRepository()
      if not os.path.exists(self.repository_path):
        parameter_list = ['clone', self.url]
        if self.branch is not None:
          parameter_list.extend(['-b', self.branch])
        parameter_list.append(self.repository_path)
        self._git(*parameter_list, cwd=self.working_directory)

155
  def checkout(self, *path_list):
156
    self.checkRepository()
157 158 159 160 161 162 163
    if not path_list:
      path_list = '.',
    revision = self.revision
    if self.getRepositoryType() == GIT_TYPE:
      # edit .git/info/sparse-checkout if you want sparse checkout
      if revision:
        if type(revision) is str:
164
          h = revision
165 166 167
        else:
          h = revision[1]
        if h != self._git('rev-parse', 'HEAD'):
168
          self.deletePycFiles(self.repository_path)
169 170 171 172 173 174
          # For performance reasons, 'reset --merge' only looks at mtime & ctime
          # to check is the index is correct and conflicts immediately if
          # contents or metadata changed. Even hardlinking a file changes its
          # ctime, so at least for buildout (local download), we need to
          # refresh index first.
          self._git('update-index', '--refresh')
175 176
          self._git('reset', '--merge', h)
      else:
177
        self.deletePycFiles(self.repository_path)
178 179 180
        if os.path.exists('.git/svn'):
          self._git('svn', 'rebase')
        else:
181 182 183 184 185
          self._git('fetch', '--all', '--prune')
          if self.branch and \
            not ("* %s" % self.branch in self._git('branch').split("\n")):
              self._git('checkout',  'origin/%s' % self.branch, '-b',
                        self.branch)
186
          self._git('update-index', '--refresh') # see note above
187
          self._git('reset', '--merge', '@{u}')
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
        self.revision = self._git_find_rev(self._git('rev-parse', 'HEAD'))
    elif self.getRepositoryType() == SVN_TYPE:
      # following code allows sparse checkout
      def svn_mkdirs(path):
        path = os.path.dirname(path)
        if path and not os.path.isdir(path):
          svn_mkdirs(path)
          self.spawn(*(args + ['--depth=empty', path]))
      for path in path_list:
        args = ['svn', 'up', '--force', '--non-interactive']
        if revision:
          args.append('-r%s' % revision)
        svn_mkdirs(path)
        args += '--set-depth=infinity', path
        self.deletePycFiles(path)
        try:
          status_dict = self.spawn(*args)
        except SubprocessError, e:
          if 'cleanup' not in e.stderr:
            raise
          self.spawn('svn', 'cleanup', path)
          status_dict = self.spawn(*args)
        if not revision:
          self.revision = revision = SVN_UP_REV.findall(
            status_dict['stdout'].splitlines()[-1])[0]
    else:
      raise NotImplementedError
    self._path_list += path_list