gitclone: Add support for cloning submodules by default

Also, add functionality to update the submodules when trying to
fetch the main repo and the reference for submodule at the main
repo has been updated.

README.rst

@@ -121,7 +121,7 @@ try to redownload resource with wrong md5sum.
  slapos.recipe.build:gitclone
 ==============================
 
-Checkout a git repository.
+Checkout a git repository and its submodules by default.
 Supports slapos.libnetworkcache if present, and if boolean 'use-cache' option
 is true.
 
@@ -391,7 +391,7 @@ Then, when update occurs, nothing is done::
   Unable to update:
   Traceback (most recent call last):
   ...
-  ...CalledProcessError: Command '['git', 'fetch', '--all']' returned non-zero exit status 1
+  ...CalledProcessError: Command '['git', 'fetch', '--recurse-submodules', '--all']' returned non-zero exit status 1
   <BLANKLINE>
   ...
   fatal: unable to access 'http://git.erp5.org/repos/nowhere/': The requested URL returned error: 500
@@ -450,6 +450,21 @@ boolean option::
   repository = https://example.net/example.git/
   ignore-ssl-certificate = true
 
+Ignore cloning submodules
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cloning the repository will clone its submodules also. You can force
+git to ignore cloinig submodules by defining `ignore-cloning-submodules` boolean
+option to 'true'::
+
+  [buildout]
+  parts = git-clone
+
+  [git-clone]
+  recipe = slapos.recipe.build:gitclone
+  repository = https://lab.nexedi.com/tiwariayush/test_erp5
+  ignore-cloning-submodules = true
+
 Other options
 ~~~~~~~~~~~~~
 

slapos/recipe/build/tests.py

@@ -8,6 +8,7 @@ import stat
 import tempfile
 import unittest
 import zc.buildout.testing
+from subprocess import check_call, check_output
 from slapos.recipe.gitclone import GIT_CLONE_ERROR_MESSAGE, \
     GIT_CLONE_CACHE_ERROR_MESSAGE
 
@@ -15,6 +16,8 @@ optionflags = (doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
 
 GIT_REPOSITORY = 'https://lab.nexedi.com/nexedi/slapos.recipe.build.git'
 BAD_GIT_REPOSITORY = 'http://git.erp5.org/repos/nowhere'
+GIT_REPOSITORY_WITH_SUBMODULES = 'https://lab.nexedi.com/tiwariayush/test_erp5'
+GIT_SUBMODULE_REPOSITORY = 'https://lab.nexedi.com/tiwariayush/test_erp5_submodule'
 REVISION = '2566127'
 
 def setUp(test):
@@ -117,7 +120,6 @@ class GitCloneNonInformativeTests(unittest.TestCase):
 
     # Check git clone parameters
     _ = self.assertIn if ignore_ssl_certificate else self.assertNotIn
-    _("--config", check_call_parameter_list[0][0])
     _("http.sslVerify=false", check_call_parameter_list[0][0])
 
     # Restore original check_call method
@@ -126,6 +128,167 @@ class GitCloneNonInformativeTests(unittest.TestCase):
   def test_ignore_ssl_certificate_false(self):
     self.test_ignore_ssl_certificate(ignore_ssl_certificate=False)
 
+  def test_clone_submodules_by_default(self, ignore_cloning_submodules=False):
+    recipe = self.makeGitCloneRecipe(
+      {'repository': GIT_REPOSITORY_WITH_SUBMODULES,
+        'ignore-cloning-submodules': str(ignore_cloning_submodules).lower()}
+    )
+    recipe.install()
+    main_repo_path = os.path.join(self.parts_directory_path, "test")
+    self.assertTrue(os.path.exists(main_repo_path))
+    submodule_repo_path = os.path.join(main_repo_path, 'dir2',
+                                       'test_erp5_submodule')
+    # Check if the folder exists
+    self.assertTrue(os.path.exists(main_repo_path))
+    # Check is there is anything in submodule repository path
+    self.assertNotEqual(bool(ignore_cloning_submodules),
+                        bool(os.listdir(submodule_repo_path)))
+
+  def test_ignore_cloning_submodules(self):
+    self.test_clone_submodules_by_default(ignore_cloning_submodules=True)
+
+  def test_fetch_submodules_with_main_repo(self):
+    """
+    Test to check the fetch of submodules while fetching main repo. Updating
+    should udpate the main repo as well as submodule repo if the reference of
+    submodule in main repo has been udpated.
+    """
+    recipe = self.makeGitCloneRecipe(
+      {'repository': GIT_REPOSITORY_WITH_SUBMODULES}
+    )
+    recipe.install()
+    main_repo_path = os.path.join(self.parts_directory_path, "test")
+    self.assertTrue(os.path.exists(main_repo_path))
+    submodule_repo_path = os.path.join(main_repo_path, 'dir2',
+                                       'test_erp5_submodule')
+    # Check if the submodule is not empty
+    self.assertTrue(os.listdir(submodule_repo_path))
+
+    # Get the head commit of the submodule repo
+    head_commit_submodule_before_fetch = check_output(['git', 'rev-parse',
+                                        'HEAD'],
+                                        cwd=submodule_repo_path).strip()
+
+    # Now, let's update the main repo as well as submodule repository.
+    # Updating the repo will fetch the updated branch but doesn't checkout
+    # to the updated master branch.
+    recipe.update()
+    head_commit_after_udpate_before_checkout = check_output(['git', 'submodule',
+                                          'status', submodule_repo_path],
+                                          cwd=main_repo_path).split()[0]
+
+    # Checkout the submodule repository and get the head commit
+    check_call(['git', 'checkout', 'master'], cwd=submodule_repo_path)
+    head_commit_after_update_after_checkout = check_output(['git', 'rev-parse',
+                                        'HEAD'],
+                                        cwd=submodule_repo_path).strip()
+
+    # Make another git clone of submodule and check the HEAD of the repo.
+    submodule_recipe = self.makeGitCloneRecipe(
+      {'repository': GIT_SUBMODULE_REPOSITORY}
+    )
+    submodule_recipe.install()
+    # Check the HEAD of the submodule
+    head_commit = check_output(['git', 'rev-parse', 'HEAD'],
+                                cwd=main_repo_path).strip()
+
+    self.assertNotEqual(head_commit_submodule_before_fetch, head_commit)
+    self.assertNotEqual(head_commit_after_udpate_before_checkout, head_commit)
+    self.assertEqual(head_commit_after_update_after_checkout, head_commit)
+
+  def test_udpate_for_submodule(self):
+    """
+    STEP I:
+    Repositories status:  Main repo(M1) and Submodule repo (S1)
+    Main repo (M1) ---references---> Submodule(S1)
+    Install should be M1+S1
+
+    STEP II:
+    Repositories status:  Main repo(M2) and Submodule repo (S2)
+    Main repo (M2) ---references---> Submodule(S1)
+    Install should be M2+S1
+
+    STEP III:
+    Repositories status:  Main repo(M3) and Submodule repo (S2)
+    Main repo (M3) ---references---> Submodule(S2)
+    Install should be M3+S2
+    """
+    # STEP I: Clone repositories in status M1 and S1 (M1---->S1)
+    recipe = self.makeGitCloneRecipe(
+      {
+        'repository': GIT_REPOSITORY_WITH_SUBMODULES,
+        'revision': '9db24704b3dece16d8bc0fbd20653018b754c0c3'
+      }
+    )
+    recipe.install()
+    main_repo_path = os.path.join(self.parts_directory_path, "test")
+    self.assertTrue(os.path.exists(main_repo_path))
+    submodule_repo_path = os.path.join(main_repo_path, 'dir2',
+                                       'test_erp5_submodule')
+    # Check if the submodule is not empty
+    self.assertTrue(os.listdir(submodule_repo_path))
+    # Get the head commit of the submodule repo
+    head_commit_submodule_after_clone = check_output(['git', 'rev-parse',
+                                        'HEAD'],
+                                        cwd=submodule_repo_path).strip()
+
+    # STEP II: Clone repositories in status M2 and S2 (M2---->S1)
+    # Update the recipe with new revision which still point to the submodule
+    # old revision
+    recipe = self.makeGitCloneRecipe(
+      {
+        'repository': GIT_REPOSITORY_WITH_SUBMODULES,
+        'revision': '69ebe9c81f7d87812b87d6fd6de45721b27520dc'
+      }
+    )
+    recipe.update()
+    self.assertTrue(os.path.exists(main_repo_path))
+    submodule_repo_path = os.path.join(main_repo_path, 'dir2',
+                                       'test_erp5_submodule')
+    # Check if the submodule is not empty
+    self.assertTrue(os.listdir(submodule_repo_path))
+    # Get the head commit of the submodule repo
+    head_commit_submodule_after_first_update = check_output(['git', 'rev-parse',
+                                        'HEAD'],
+                                        cwd=submodule_repo_path).strip()
+
+    # STEP II: Clone repositories in status M3 and S2 (M3---->S2)
+    # Update the recipe with new revision which points to submodule new revision
+    recipe = self.makeGitCloneRecipe(
+      {
+        'repository': GIT_REPOSITORY_WITH_SUBMODULES,
+        'revision': '05a26f99b2d366830e50088e90f94708fdd69f61'
+      }
+    )
+    recipe.update()
+    self.assertTrue(os.path.exists(main_repo_path))
+    submodule_repo_path = os.path.join(main_repo_path, 'dir2',
+                                       'test_erp5_submodule')
+    # Check if the submodule is not empty
+    self.assertTrue(os.listdir(submodule_repo_path))
+    # Get the head commit of the submodule repo
+    head_commit_submodule_after_second_update = check_output(['git', 'rev-parse',
+                                        'HEAD'],
+                                        cwd=submodule_repo_path).strip()
+
+    # Make another git clone of submodule and check the HEAD of the repo.
+    submodule_recipe = self.makeGitCloneRecipe(
+      {'repository': GIT_SUBMODULE_REPOSITORY}
+    )
+    submodule_recipe.install()
+    # Check the HEAD of the submodule
+    head_commit = check_output(['git', 'rev-parse', 'HEAD'],
+                                cwd=main_repo_path).strip()
+
+    self.assertEqual(head_commit_submodule_after_clone,
+                     head_commit_submodule_after_first_update)
+
+    self.assertNotEqual(head_commit_submodule_after_first_update,
+                        head_commit_submodule_after_second_update)
+
+    self.assertNotEqual(head_commit_submodule_after_second_update,
+                        head_commit)
+
 def test_suite():
   suite = unittest.TestSuite((
       doctest.DocFileSuite(

slapos/recipe/gitclone.py

@@ -145,7 +145,8 @@ class Recipe(object):
       self.git_command = 'git'
     self.sparse = options.get('sparse-checkout', '').strip()
     # Set boolean values
-    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate'):
+    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate',
+                'ignore-cloning-submodules'):
       setattr(self, key.replace('-', '_'), options.get(key, '').lower() in TRUE_VALUES)
     if self.shared:
       self.use_cache = False
@@ -213,6 +214,13 @@ class Recipe(object):
 
     if config and self.use_cache:
       raise NotImplementedError
+
+    if not self.ignore_cloning_submodules:
+      # `--recurse-submodules` to the git clone command will automatically
+      # initialize and update each submodule in the repository.
+      config.append('submodule.recurse=true')
+      git_clone_command.append('--recurse-submodules')
+
     for config in config:
       git_clone_command += '--config', config
 
@@ -274,7 +282,15 @@ class Recipe(object):
             call([self.git_command, 'rev-parse', '--verify', self.revision],
                     cwd=self.location) == 0
       if not revision_already_fetched:
-        check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
+        # With --recurse-submodules, it will fetch the updated branches of the
+        # submodules as well as of the main repo. It will remain in a detached
+        # head old state
+        # NOTE: It won't checkout to the updated master/main-branch
+        check_call([self.git_command, 'fetch', '--recurse-submodules', '--all'],
+                    cwd=self.location)
+        # Also update the submodule directory
+        check_call([self.git_command, 'submodule', 'update', '--remote'],
+                    cwd=self.location)
 
       # If develop parameter is set, don't reset/update.
       # Otherwise, reset --hard