Commit 5e95d882 authored by Ayush Tiwari's avatar Ayush Tiwari Committed by Julien Muchembled

gitclone: add support for submodules, enabled by default

An unrelated change is about develop mode: the behaviour was simplified to do nothing if a working copy already exists.

TODO: handle removal of submodules

/reviewed-on nexedi/slapos.recipe.build!7
parent 5e6da988
......@@ -121,7 +121,7 @@ try to redownload resource with wrong md5sum.
slapos.recipe.build:gitclone
==============================
Checkout a git repository.
Checkout a git repository and its submodules by default.
Supports slapos.libnetworkcache if present, and if boolean 'use-cache' option
is true.
......@@ -236,6 +236,7 @@ Then let's run the buildout::
Running uninstall recipe.
Installing git-clone.
Cloning into '/sample-buildout/parts/git-clone'...
HEAD is now at 2566127 ...
Let's take a look at the buildout parts directory now::
......@@ -249,15 +250,12 @@ And let's see that current revision is "2566127"::
>>> print(subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], universal_newlines=True))
2566127
When updating, it will do a "git fetch; git reset revision"::
When updating, it shouldn't do anything as revision is mentioned::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
...
HEAD is now at 2566127 ...
...
<BLANKLINE>
Empty revision/branch
~~~~~~~~~~~~~~~~~~~~~
......@@ -327,7 +325,7 @@ and branch parameter is ignored::
Setup a "develop" repository
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you need to setup a repository that will be manually alterated over time for
If you need to setup a repository that will be manually altered over time for
development purposes, you need to make sure buildout will NOT alter it and NOT
erase your local modifications by specifying the "develop" flag::
......@@ -366,7 +364,6 @@ repository::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
Fetching origin
...
<BLANKLINE>
......@@ -386,17 +383,7 @@ Then, when update occurs, nothing is done::
>>> cd(sample_buildout)
>>> print(system(buildout))
Updating git-clone.
Fetching origin
Fetching broken
Unable to update:
Traceback (most recent call last):
...
...CalledProcessError: Command '['git', 'fetch', '--all']' returned non-zero exit status 1
<BLANKLINE>
...
fatal: unable to access 'http://git.erp5.org/repos/nowhere/': The requested URL returned error: 500
error: Could not fetch broken
<BLANKLINE>
>>> cd(sample_buildout, 'parts', 'git-clone')
>>> print(system('cat local_change'))
......@@ -450,6 +437,21 @@ boolean option::
repository = https://example.net/example.git/
ignore-ssl-certificate = true
Ignore cloning submodules
~~~~~~~~~~~~~~~~~~~~~~~~~
By default, cloning the repository will clone its submodules also. You can force
git to ignore cloning submodules by defining `ignore-cloning-submodules` boolean
option to 'true'::
[buildout]
parts = git-clone
[git-clone]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/tiwariayush/test_erp5
ignore-cloning-submodules = true
Other options
~~~~~~~~~~~~~
......
......@@ -4,10 +4,13 @@ from zope.testing import renormalizing
import os
import re
import shutil
import stat
import tempfile
import unittest
import zc.buildout.testing
from zc.buildout.testing import buildoutTearDown
from contextlib import contextmanager
from functools import wraps
from subprocess import check_call, check_output, CalledProcessError
from slapos.recipe.gitclone import GIT_CLONE_ERROR_MESSAGE, \
GIT_CLONE_CACHE_ERROR_MESSAGE
......@@ -26,25 +29,134 @@ def setUp(test):
zc.buildout.testing.buildoutSetUp(test)
zc.buildout.testing.install_develop('slapos.recipe.build', test)
@contextmanager
def chdir(path):
old = os.getcwd()
try:
os.chdir(path)
yield old
finally:
os.chdir(old)
def with_buildout(wrapped):
def wrapper(self):
self.globs = {}
setUp(self)
try:
wrapped(self, **self.globs)
finally:
buildoutTearDown(self)
return wraps(wrapped)(wrapper)
class GitCloneNonInformativeTests(unittest.TestCase):
def setUp(self):
self.dir = os.path.realpath(tempfile.mkdtemp())
self.parts_directory_path = os.path.join(self.dir, 'test_parts')
def setUpParentRepository(self):
"""
This function sets ups repositories for parent repo and adds file and commit
to it.
"""
# Create parent and submodule directory
self.project_dir = os.path.join(self.dir, 'main_repo')
os.mkdir(self.project_dir)
# Add files and folder in parent repo and initialize git in it
check_call(['git', 'init'], cwd=self.project_dir)
self.setUpGitConfig(self.project_dir)
os.mkdir(os.path.join(self.project_dir, 'dir1'))
self.touch(self.project_dir, 'file1.py')
self.gitAdd(self.project_dir)
self.gitCommit(self.project_dir, msg='Add file and folder in main repo')
def setUpSubmoduleRepository(self):
"""
This function sets up repositories and config for parent repo and a
submodule repo and create small commits as well as links the 2 repos.
"""
# Create parent and submodule directory
self.submodule_dir = os.path.join(self.dir ,'submodule_repo')
os.mkdir(self.submodule_dir)
# Add files in submodule repo and initialize git in it
check_call(['git', 'init'], cwd=self.submodule_dir)
self.setUpGitConfig(self.submodule_dir)
self.touch(self.submodule_dir, 'file1.py')
self.gitAdd(self.submodule_dir)
self.gitCommit(self.submodule_dir, msg='Add file1 in submodule repo')
def attachSubmoduleToParent(self):
"""
Adds the submodule repo to parent repo and creates a commit in parent if
parent and submodule repo are present.
"""
assert hasattr(self, 'project_dir') and hasattr(self, 'submodule_dir'), (
"Make sure parent repo and submodule repo are present")
submodule_dir_main_repo = os.path.join(self.project_dir, 'dir1')
# Add submodule to main repo and commit
check_call(['git', 'submodule', 'add', self.submodule_dir],
cwd=submodule_dir_main_repo)
self.gitCommit(self.project_dir, msg='Add submodule repo')
def createRepositoriesAndConnect(self):
"""
Creates parent and submodule repository and then adds the submodule repo to
parent repo and creates a commit in parent.
"""
self.setUpParentRepository()
self.setUpSubmoduleRepository()
self.attachSubmoduleToParent()
def tearDown(self):
shutil.rmtree(self.dir)
for var in list(os.environ):
if var.startswith('SRB_'):
del os.environ[var]
def write_file(self, filename, contents, mode=stat.S_IREAD | stat.S_IWUSR):
path = os.path.join(self.dir, filename)
fh = open(path, 'w')
fh.write(contents)
fh.close()
os.chmod(path, mode)
return path
def setUpGitConfig(self, proj_dir):
"""
Setup user and email for given git repository
"""
check_call(['git', 'config', 'user.email', 'test@example.com'], cwd=proj_dir)
check_call(['git', 'config', 'user.name', 'Test'], cwd=proj_dir)
def gitOutput(self, *args, **kw):
return check_output(('git',) + args, universal_newlines=True, **kw)
def gitAdd(self, proj_dir):
"""runs a 'git add .' in the provided git repo
:param proj_dir: path to a git repo
"""
check_call(['git', 'add', '.'], cwd=proj_dir)
def gitCommit(self, proj_dir, msg):
"""runs a 'git commit -m msg' in the provided git repo
:param proj_dir: path to a git repo
"""
check_call(['git', 'commit', '-m', msg], cwd=proj_dir)
def getRepositoryHeadCommit(self, proj_dir):
"""
Returns the sha of HEAD of a git repo
"""
return self.gitOutput('rev-parse', 'HEAD', cwd=proj_dir).rstrip()
def touch(self, *parts):
os.close(os.open(os.path.join(*parts), os.O_CREAT, 0o666))
def readFile(self, *parts):
with open(os.path.join(*parts), 'r') as f:
return f.read()
def checkLocalChanges(self, parent_local_change_path,
submodule_local_change_path):
# Check if the file are created at the expected position and check contents
self.assertTrue(os.path.exists(parent_local_change_path))
self.assertTrue(os.path.exists(submodule_local_change_path))
self.assertEqual(self.readFile(parent_local_change_path), 'foo')
self.assertEqual(self.readFile(submodule_local_change_path), 'bar')
def makeGitCloneRecipe(self, options):
from slapos.recipe.gitclone import Recipe
......@@ -63,15 +175,15 @@ class GitCloneNonInformativeTests(unittest.TestCase):
def test_using_download_cache_if_git_fails(self):
recipe = self.makeGitCloneRecipe({"use-cache": "true",
"repository": BAD_GIT_REPOSITORY})
os.chdir(self.dir)
with self.assertRaises(zc.buildout.UserError) as cm:
with chdir(self.dir), \
self.assertRaises(zc.buildout.UserError) as cm:
recipe.install()
self.assertEqual(str(cm.exception), GIT_CLONE_CACHE_ERROR_MESSAGE)
def test_not_using_download_cache_if_forbidden(self):
recipe = self.makeGitCloneRecipe({"repository": BAD_GIT_REPOSITORY})
os.chdir(self.dir)
with self.assertRaises(zc.buildout.UserError) as cm:
with chdir(self.dir), \
self.assertRaises(zc.buildout.UserError) as cm:
recipe.install()
self.assertEqual(str(cm.exception), GIT_CLONE_ERROR_MESSAGE)
......@@ -90,6 +202,239 @@ class GitCloneNonInformativeTests(unittest.TestCase):
self.assertTrue(os.path.exists(git_repository_path))
self.assertFalse(os.path.exists(bad_file_path), "pyc file not removed")
@with_buildout
def test_clone_and_update_submodule(self, buildout, write,
sample_buildout, **kw):
"""
Remote:
Repositories status: Parent repo(M1) and Submodule repo (S1)
Parent repo (M1) ---references---> Submodule(S1)
Local:
Buildout should install(using branch) at M1+S1
Remote:
Repositories status: Parent repo(M2) and Submodule repo (S2)
Parent repo (M2) ---references---> Submodule(S1)
Local:
Buildout should update to at M2+S1
Remote:
Repositories status: Parent repo(M3) and Submodule repo (S2)
Parent repo (M3) ---references---> Submodule(S2)
Local:
Buildout should update to M3+S2
"""
self.createRepositoriesAndConnect()
# Clone repositories in status M1 and S1 (M1---->S1)
write(sample_buildout, 'buildout.cfg',
"""
[buildout]
parts = git-clone
[git-clone]
recipe = slapos.recipe.build:gitclone
repository = %s
""" % self.project_dir)
check_call([buildout])
main_repo_path = os.path.join(sample_buildout, 'parts', 'git-clone')
self.assertTrue(os.path.exists(main_repo_path))
submodule_repo_path = os.path.join(main_repo_path, 'dir1',
'submodule_repo')
# Check if the submodule is not empty
self.assertTrue(os.listdir(submodule_repo_path))
# Get the head commit of the submodule repo
head_commit_submodule_after_clone = self.getRepositoryHeadCommit(
submodule_repo_path)
# Add untracked files as markers to check that the part
# was updated rather than removed+reinstalled.
write(main_repo_path, 'local_change_main', 'foo')
write(submodule_repo_path, 'local_change_submodule', 'bar')
parent_local_change_path = os.path.join(main_repo_path,
'local_change_main')
submodule_local_change_path = os.path.join(submodule_repo_path,
'local_change_submodule')
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
# Trigger `update` method call for gitclone recipe
check_call([buildout])
# The local changes should be still there after update
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
# On REMOTE, update submodule repository and parent repo with new commit,
# but do not update the pointer to submodule on parent repo
self.touch(self.project_dir, 'file2.py')
self.gitAdd(self.project_dir)
self.gitCommit(self.project_dir, msg='Add file2 in main repo')
self.touch(self.submodule_dir, 'file2.py')
self.gitAdd(self.submodule_dir)
self.gitCommit(self.submodule_dir, msg='Add file2 in submodule repo')
# Trigger update on the same branch. Remember the state at remote is
# M2 and S2 (M2---->S1)
check_call([buildout])
head_commit_submodule_after_first_update = self.getRepositoryHeadCommit(
submodule_repo_path)
# The local changes should be still there after update
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
# On REMOTE, add new commit to submodule and then update the submodule
# pointer on parent repo and commit it
submodule_dir_main_repo = os.path.join(self.project_dir, 'dir1',
'submodule_repo')
check_call(['git', 'checkout', 'master'], cwd=submodule_dir_main_repo)
check_call(['git', 'pull', '--ff'], cwd=submodule_dir_main_repo)
self.gitAdd(self.project_dir)
self.gitCommit(self.project_dir, msg='Update submodule version')
# Trigger update again on the same branch. Remember the state at remote is
# M3 and S2(M3---->S2)
check_call([buildout])
head_commit_submodule_after_second_update = self.getRepositoryHeadCommit(
submodule_repo_path)
# The local changes should be still there after update
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
# Check the HEAD of the submodule
submodule_head_commit = self.getRepositoryHeadCommit(self.submodule_dir)
self.assertEqual(head_commit_submodule_after_clone,
head_commit_submodule_after_first_update)
self.assertNotEqual(head_commit_submodule_after_first_update,
head_commit_submodule_after_second_update)
self.assertEqual(head_commit_submodule_after_second_update,
submodule_head_commit)
# Update the recipe with new revision for parent and trigger uninstall/
# install
write(sample_buildout, 'buildout.cfg',
"""
[buildout]
parts = git-clone
[git-clone]
recipe = slapos.recipe.build:gitclone
repository = %s
revision = %s
""" % (self.project_dir, str(self.getRepositoryHeadCommit(self.project_dir))))
check_call([buildout])
self.assertTrue(os.path.exists(main_repo_path))
# Check if the submodule is not empty
self.assertTrue(os.listdir(submodule_repo_path))
# Since calling buildout should've reinstalled, we expect the local changes
# to be gone
self.assertFalse(os.path.exists(parent_local_change_path))
self.assertFalse(os.path.exists(submodule_local_change_path))
# Get the head commit of the submodule repo
head_commit_submodule_with_revision = self.getRepositoryHeadCommit(
submodule_repo_path)
self.assertEqual(head_commit_submodule_with_revision,
submodule_head_commit)
@with_buildout
def test_clone_install_and_udpate_develop_mode(self, buildout, write,
sample_buildout, **kw):
"""
Test to verify the result of development mode, i.e., develop = True.
In this case, we expect local changes to be untouched
"""
self.createRepositoriesAndConnect()
# Clone repositories in status M1 and S1 (M1---->S1)
write(sample_buildout, 'buildout.cfg',
"""
[buildout]
parts = git-clone
[git-clone]
recipe = slapos.recipe.build:gitclone
repository = %s
develop=True
""" % self.project_dir)
check_call([buildout])
main_repo_path = os.path.join(sample_buildout, 'parts', 'git-clone')
self.assertTrue(os.path.exists(main_repo_path))
submodule_repo_path = os.path.join(main_repo_path, 'dir1',
'submodule_repo')
# Check if the submodule is not empty
self.assertTrue(os.listdir(submodule_repo_path))
# Add untracked files as markers to check that the part
# was updated rather than removed+reinstalled.
write(main_repo_path, 'local_change_main', 'foo')
write(submodule_repo_path, 'local_change_submodule', 'bar')
parent_local_change_path = os.path.join(main_repo_path,
'local_change_main')
submodule_local_change_path = os.path.join(submodule_repo_path,
'local_change_submodule')
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
# Trigger `update` method call for gitclone recipe
check_call([buildout])
# The local changes should be still there after update as it is develop mode
self.checkLocalChanges(parent_local_change_path, submodule_local_change_path)
@with_buildout
def test_git_add_for_submodule_changes(self, buildout, write,
sample_buildout, **kw):
"""
Test to verify the result of `git status` being executed from parent as well
as submodule repository after making some local changes in submodule repo
"""
# Create a parent repo
self.setUpParentRepository()
# Create submodule repository
self.setUpSubmoduleRepository()
# Attach the submodule repository to the parent repo on remote
self.attachSubmoduleToParent()
write(sample_buildout, 'buildout.cfg',
"""
[buildout]
parts = git-clone
[git-clone]
recipe = slapos.recipe.build:gitclone
repository = %s
""" % self.project_dir)
check_call([buildout])
local_parent_dir = os.path.join(sample_buildout, 'parts', 'git-clone')
local_submodule_dir = os.path.join(local_parent_dir, 'dir1',
'submodule_repo')
# Now so some change manually in submodule repo, but don't commit
self.touch(local_submodule_dir, 'file2.py')
self.gitAdd(local_submodule_dir)
# Do `git status` to check if the changes are shown for the repo in parent
# This should the changes been done in the main as well as submodule repo
files_changed = self.gitOutput('status', '--porcelain',
cwd=local_parent_dir)
file_changed, = files_changed.splitlines()
# Check if submodule directory is part of modified list
self.assertEqual(file_changed, ' M dir1/submodule_repo')
# Now that `git status` in parent repo shows changes in the submodule repo,
# do `git status` in the submodule repo to re-confirm the exact file change
files_changed = self.gitOutput('status', '--porcelain',
cwd=local_submodule_dir)
file_changed, = files_changed.splitlines()
# Check if submodule directory is part of modified list
self.assertEqual(file_changed, 'A file2.py')
def test_ignore_ssl_certificate(self, ignore_ssl_certificate=True):
import slapos.recipe.gitclone
......@@ -117,7 +462,6 @@ class GitCloneNonInformativeTests(unittest.TestCase):
# Check git clone parameters
_ = self.assertIn if ignore_ssl_certificate else self.assertNotIn
_("--config", check_call_parameter_list[0][0])
_("http.sslVerify=false", check_call_parameter_list[0][0])
# Restore original check_call method
......@@ -126,6 +470,26 @@ class GitCloneNonInformativeTests(unittest.TestCase):
def test_ignore_ssl_certificate_false(self):
self.test_ignore_ssl_certificate(ignore_ssl_certificate=False)
def test_clone_submodules_by_default(self, ignore_cloning_submodules=False):
self.createRepositoriesAndConnect()
recipe = self.makeGitCloneRecipe(
{'repository': self.project_dir,
'ignore-cloning-submodules': str(ignore_cloning_submodules).lower()}
)
recipe.install()
main_repo_path = os.path.join(self.parts_directory_path, "test")
self.assertTrue(os.path.exists(main_repo_path))
submodule_repo_path = os.path.join(main_repo_path, 'dir1',
'submodule_repo')
# Check if the folder exists
self.assertTrue(os.path.exists(main_repo_path))
# Check is there is anything in submodule repository path
self.assertNotEqual(bool(ignore_cloning_submodules),
bool(os.listdir(submodule_repo_path)))
def test_ignore_cloning_submodules(self):
self.test_clone_submodules_by_default(ignore_cloning_submodules=True)
def test_suite():
suite = unittest.TestSuite((
doctest.DocFileSuite(
......
......@@ -34,7 +34,7 @@ import time
import traceback
from zc.buildout import UserError
from subprocess import call, check_call, CalledProcessError
from subprocess import check_call, CalledProcessError
import subprocess
try:
......@@ -145,7 +145,8 @@ class Recipe(object):
self.git_command = 'git'
self.sparse = options.get('sparse-checkout', '').strip()
# Set boolean values
for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate'):
for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate',
'ignore-cloning-submodules'):
setattr(self, key.replace('-', '_'), options.get(key, '').lower() in TRUE_VALUES)
if self.shared:
self.use_cache = False
......@@ -171,7 +172,6 @@ class Recipe(object):
command.append(revision)
check_call(command, cwd=self.location)
def install(self):
"""
Do a git clone.
......@@ -213,6 +213,13 @@ class Recipe(object):
if config and self.use_cache:
raise NotImplementedError
if not self.ignore_cloning_submodules:
# `--recurse-submodules` to the git clone command will automatically
# initialize and update each submodule in the repository.
config.append('submodule.recurse=true')
git_clone_command.append('--recurse-submodules')
for config in config:
git_clone_command += '--config', config
......@@ -230,6 +237,11 @@ class Recipe(object):
if self.use_cache:
upload_network_cached(os.path.join(self.location, '.git'),
self.repository, self.revision, self.networkcache)
if not self.ignore_cloning_submodules:
# Update submodule repository to the commit which is being pointed to
# in main repo
check_call([self.git_command, 'submodule', 'update', '--init',
'--recursive'], cwd=self.location)
except CalledProcessError:
print("Unable to download from git repository."
" Trying from network cache...")
......@@ -260,32 +272,35 @@ class Recipe(object):
def update(self):
"""
Do a git fetch.
If user doesn't develop, reset to remote revision (or branch if revision is
not specified).
If user doesn't develop, reset to upstream of the branch.
With support for submodules, we also update the submodule repository if
there has been update to the pointer of the submodules in the parent repo.
This however puts the updated submodules in a DETACHED state.
"""
# If develop or revision parameter, no need to update
if self.develop or self.revision:
return
try:
# first cleanup pyc files
self.deletePycFiles(self.location)
# then update,
# but, to save time, only if we don't have the revision already
revision_already_fetched = \
self.revision and \
call([self.git_command, 'rev-parse', '--verify', self.revision],
cwd=self.location) == 0
if not revision_already_fetched:
check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
# Fetch and reset to the upstream
check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
self.gitReset('@{upstream}')
if not self.ignore_cloning_submodules:
# Update the submodule to the commit which parent repo points to if
# there has been revision is present for the parent repo.
# According to man-page of submodule, update also clones missing
# submodules and updating the working tree of the submodules. This is
# why we do update here only after we are ok with revision of parent
# repo being checked out to the desired one.
# It will also init a submodule if required
# NOTE: This will put the submodule repo in a `Detached` state.
check_call([self.git_command, 'submodule', 'update', '--init', '-f',
'--recursive'], cwd=self.location)
# If develop parameter is set, don't reset/update.
# Otherwise, reset --hard
if not self.develop:
if self.revision:
self.gitReset(self.revision)
else:
self.gitReset('@{upstream}')
except:
if not self.develop:
raise
# Buildout will remove the installed location and mark the part as not
# installed if an error occurs during update. If we are developping this
# repository we do not want this to happen.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment