gitclone: Add support for cloning submodules by default

Also, add functionality to fetch the submodules while fetching the
the main repo and checkout the submodule(s) repos in case the pointer
to the submodule in the main repo has been updated.

README.rst

@@ -121,7 +121,7 @@ try to redownload resource with wrong md5sum.
  slapos.recipe.build:gitclone
 ==============================
 
-Checkout a git repository.
+Checkout a git repository and its submodules by default.
 Supports slapos.libnetworkcache if present, and if boolean 'use-cache' option
 is true.
 
@@ -391,7 +391,7 @@ Then, when update occurs, nothing is done::
   Unable to update:
   Traceback (most recent call last):
   ...
-  ...CalledProcessError: Command '['git', 'fetch', '--all']' returned non-zero exit status 1
+  ...CalledProcessError: Command '['git', 'fetch', '--recurse-submodules', '--all']' returned non-zero exit status 1
   <BLANKLINE>
   ...
   fatal: unable to access 'http://git.erp5.org/repos/nowhere/': The requested URL returned error: 500
@@ -450,6 +450,21 @@ boolean option::
   repository = https://example.net/example.git/
   ignore-ssl-certificate = true
 
+Ignore cloning submodules
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, cloning the repository will clone its submodules also. You can force
+git to ignore cloinig submodules by defining `ignore-cloning-submodules` boolean
+option to 'true'::
+
+  [buildout]
+  parts = git-clone
+
+  [git-clone]
+  recipe = slapos.recipe.build:gitclone
+  repository = https://lab.nexedi.com/tiwariayush/test_erp5
+  ignore-cloning-submodules = true
+
 Other options
 ~~~~~~~~~~~~~
 

slapos/recipe/gitclone.py

@@ -34,7 +34,7 @@ import time
 import traceback
 
 from zc.buildout import UserError
-from subprocess import call, check_call, CalledProcessError
+from subprocess import call, check_call, check_output, CalledProcessError
 import subprocess
 
 try:
@@ -145,7 +145,8 @@ class Recipe(object):
       self.git_command = 'git'
     self.sparse = options.get('sparse-checkout', '').strip()
     # Set boolean values
-    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate'):
+    for key in ('develop', 'shared', 'use-cache', 'ignore-ssl-certificate',
+                'ignore-cloning-submodules'):
       setattr(self, key.replace('-', '_'), options.get(key, '').lower() in TRUE_VALUES)
     if self.shared:
       self.use_cache = False
@@ -213,6 +214,13 @@ class Recipe(object):
 
     if config and self.use_cache:
       raise NotImplementedError
+
+    if not self.ignore_cloning_submodules:
+      # `--recurse-submodules` to the git clone command will automatically
+      # initialize and update each submodule in the repository.
+      config.append('submodule.recurse=true')
+      git_clone_command.append('--recurse-submodules')
+
     for config in config:
       git_clone_command += '--config', config
 
@@ -230,6 +238,9 @@ class Recipe(object):
       if self.use_cache:
         upload_network_cached(os.path.join(self.location, '.git'),
                               self.repository, self.revision, self.networkcache)
+      if not self.ignore_cloning_submodules:
+        check_call([self.git_command, 'submodule', 'update', '--recursive'],
+                    cwd=self.location)
     except CalledProcessError:
       print("Unable to download from git repository."
             " Trying from network cache...")
@@ -262,6 +273,9 @@ class Recipe(object):
     Do a git fetch.
     If user doesn't develop, reset to remote revision (or branch if revision is
     not specified).
+    With support for submodules, we also update the submodule repository if
+    there has been update to the pointer of the submodules in the parent repo.
+    This however puts the updated submodules in a DETACHED state.
     """
     try:
       # first cleanup pyc files
@@ -272,9 +286,15 @@ class Recipe(object):
       revision_already_fetched = \
             self.revision and   \
             call([self.git_command, 'rev-parse', '--verify', self.revision],
-                    cwd=self.location) == 0
+                    cwd=self.location) == 0 and \
+            check_output([self.git_command, 'rev-parse', 'HEAD'],
+                    cwd=self.location).strip() == self.revision
+
       if not revision_already_fetched:
-        check_call([self.git_command, 'fetch', '--all'], cwd=self.location)
+        # With --recurse-submodules, it will fetch the updated branches of the
+        # submodules as well as of the main repo.
+        check_call([self.git_command, 'fetch', '--recurse-submodules', '--all'],
+                    cwd=self.location)
 
       # If develop parameter is set, don't reset/update.
       # Otherwise, reset --hard
@@ -283,6 +303,14 @@ class Recipe(object):
           self.gitReset(self.revision)
         else:
           self.gitReset('@{upstream}')
+
+      if not revision_already_fetched:
+        # Also update the submodule directory to the commit which parent
+        # repository is pointing to.
+        # NOTE: This will put the submodule repo in a `Detached` state.
+        check_call([self.git_command, 'submodule', 'update', '--recursive'],
+                    cwd=self.location)
+
     except:
       if not self.develop:
         raise