Merge branch '62418-project-default-git-depth' into 'master'

Add project level git depth setting

Closes #59688

See merge request gitlab-org/gitlab-ce!28919

app/controllers/projects/settings/ci_cd_controller.rb

@@ -50,7 +50,8 @@ module Projects
           :runners_token, :builds_enabled, :build_allow_git_fetch,
           :build_timeout_human_readable, :build_coverage_regex, :public_builds,
           :auto_cancel_pending_pipelines, :ci_config_path,
-          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy]
+          auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy],
+          ci_cd_settings_attributes: [:default_git_depth]
         )
       end
 

app/models/project.rb

@@ -292,6 +292,7 @@ class Project < ApplicationRecord
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
+  accepts_nested_attributes_for :ci_cd_settings, update_only: true
 
   accepts_nested_attributes_for :remote_mirrors,
                                 allow_destroy: true,
@@ -310,6 +311,7 @@ class Project < ApplicationRecord
   delegate :root_ancestor, to: :namespace, allow_nil: true
   delegate :last_pipeline, to: :commit, allow_nil: true
   delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true
+  delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings
 
   # Validations
   validates :creator, presence: true, on: :create

app/models/project_ci_cd_setting.rb

@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord
   # The version of the schema that first introduced this model/table.
   MINIMUM_SCHEMA_VERSION = 20180403035759
 
+  DEFAULT_GIT_DEPTH = 50
+
+  before_create :set_default_git_depth
+
+  validates :default_git_depth,
+    numericality: {
+      only_integer: true,
+      greater_than_or_equal_to: 0,
+      less_than_or_equal_to: 1000
+    },
+    allow_nil: true
+
   def self.available?
     @available ||=
       ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION
@@ -15,4 +27,10 @@ class ProjectCiCdSetting < ApplicationRecord
     @available = nil
     super
   end
+
+  private
+
+  def set_default_git_depth
+    self.default_git_depth ||= DEFAULT_GIT_DEPTH
+  end
 end

app/presenters/ci/build_runner_presenter.rb

@@ -25,14 +25,16 @@ module Ci
     end
 
     def git_depth
-      strong_memoize(:git_depth) do
-        git_depth = variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }&.dig(:value)
-        git_depth.to_i
-      end
+      if git_depth_variable
+        git_depth_variable[:value]
+      else
+        project.default_git_depth
+      end.to_i
     end
 
     def refspecs
       specs = []
+      specs << refspec_for_merge_request_ref if merge_request_ref?
 
       if git_depth > 0
         specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
@@ -42,8 +44,6 @@ module Ci
         specs << refspec_for_tag
       end
 
-      specs << refspec_for_merge_request_ref if merge_request_ref?
-
       specs
     end
 
@@ -89,5 +89,11 @@ module Ci
     def refspec_for_merge_request_ref
       "+#{ref}:#{ref}"
     end
+
+    def git_depth_variable
+      strong_memoize(:git_depth_variable) do
+        variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }
+      end
+    end
   end
 end

app/services/projects/fork_service.rb

@@ -43,6 +43,10 @@ module Projects
         shared_runners_enabled:    @project.shared_runners_enabled,
         namespace_id:              target_namespace.id,
         fork_network:              fork_network,
+        # We need to set default_git_depth to 0 for the forked project when
+        # @project.default_git_depth is nil in order to keep the same behaviour
+        # and not get ProjectCiCdSetting::DEFAULT_GIT_DEPTH set on create
+        ci_cd_settings_attributes: { default_git_depth: @project.default_git_depth || 0 },
         # We need to assign the fork network membership after the project has
         # been instantiated to avoid ActiveRecord trying to create it when
         # initializing the project, as that would cause a foreign key constraint

app/views/projects/settings/ci_cd/_form.html.haml

@@ -24,6 +24,14 @@
               %span.descr
                 = _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)")
 
+        %hr
+        .form-group
+          = f.fields_for :ci_cd_settings_attributes, @project.ci_cd_settings do |form|
+            = form.label :default_git_depth, _('Git shallow clone'), class: 'label-bold'
+            = form.number_field :default_git_depth, { class: 'form-control', min: 0, max: 1000 }
+            %p.form-text.text-muted
+              = _('The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time.')
+
         %hr
         .form-group
           = f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold'

changelogs/unreleased/62418-project-default-git-depth.yml

+---
+title: Add project level git depth CI/CD setting
+merge_request: 28919
+author:
+type: added

db/migrate/20190530042141_add_default_git_depth_to_ci_cd_settings.rb

+# frozen_string_literal: true
+
+class AddDefaultGitDepthToCiCdSettings < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :project_ci_cd_settings, :default_git_depth, :integer
+  end
+end

db/schema.rb

@@ -1660,6 +1660,7 @@ ActiveRecord::Schema.define(version: 20190530154715) do
     t.boolean "group_runners_enabled", default: true, null: false
     t.boolean "merge_pipelines_enabled"
     t.boolean "merge_trains_enabled", default: false, null: false
+    t.integer "default_git_depth"
     t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
   end
 

doc/user/project/pipelines/settings.md

@@ -20,6 +20,22 @@ There are two options. Using:
 The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy)
 in `.gitlab-ci.yml`.
 
+## Git shallow clone
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28919) in GitLab 12.0.
+
+NOTE: **Note**: As of GitLab 12.0, newly created projects will automaticallyl have a default 
+`git depth` value of `50`.
+
+It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning 
+a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum
+allowed value is `1000`.
+
+To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time,
+keep the value empty or set to `0`.
+
+This value can also be [overridden by `GIT_DEPTH`](../../../ci/large_repositories/index.md#shallow-cloning) variable in `.gitlab-ci.yml` file.
+
 ## Timeout
 
 Timeout defines the maximum amount of time in minutes that a job is able run.

locale/gitlab.pot

@@ -4668,6 +4668,9 @@ msgstr ""
 msgid "Git revision"
 msgstr ""
 
+msgid "Git shallow clone"
+msgstr ""
+
 msgid "Git strategy for pipelines"
 msgstr ""
 
@@ -9969,6 +9972,9 @@ msgstr ""
 msgid "The name %{entryName} is already taken in this directory."
 msgstr ""
 
+msgid "The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time."
+msgstr ""
+
 msgid "The number of times an upload record could not find its file"
 msgstr ""
 

spec/controllers/projects/settings/ci_cd_controller_spec.rb

@@ -200,6 +200,21 @@ describe Projects::Settings::CiCdController do
           expect(response).to redirect_to(namespace_project_settings_ci_cd_path)
         end
       end
+
+      context 'when default_git_depth is not specified' do
+        let(:params) { { ci_cd_settings_attributes: { default_git_depth: 10 } } }
+
+        before do
+          project.ci_cd_settings.update!(default_git_depth: nil)
+        end
+
+        it 'set specified git depth' do
+          subject
+
+          project.reload
+          expect(project.default_git_depth).to eq(10)
+        end
+      end
     end
   end
 end

spec/lib/gitlab/background_migration/delete_diff_files_spec.rb

@@ -3,6 +3,12 @@ require 'spec_helper'
 # rubocop:disable RSpec/FactoriesInMigrationSpecs
 describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do
   describe '#perform' do
+    before do
+      # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
+      allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    end
+
     context 'when diff files can be deleted' do
       let(:merge_request) { create(:merge_request, :merged) }
       let!(:merge_request_diff) do

spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb

@@ -9,6 +9,9 @@ describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration
   before do
     # This migration was created before we introduced metadata configs
     stub_feature_flags(ci_build_metadata_config: false)
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
   end
 
   let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }

spec/migrations/remove_orphaned_label_links_spec.rb

@@ -10,6 +10,12 @@ describe RemoveOrphanedLabelLinks, :migration do
   let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
   let(:label) { create_label }
 
+  before do
+    # This migration was created before we introduced ProjectCiCdSetting#default_git_depth
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
+    allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth=).and_return(0)
+  end
+
   context 'add foreign key on label_id' do
     let!(:label_link_with_label) { create_label_link(label_id: label.id) }
     let!(:label_link_without_label) { create_label_link(label_id: nil) }

spec/models/project_ci_cd_setting_spec.rb

@@ -21,4 +21,32 @@ describe ProjectCiCdSetting do
       2.times { described_class.available? }
     end
   end
+
+  describe 'validations' do
+    it 'validates default_git_depth is between 0 and 1000 or nil' do
+      expect(subject).to validate_numericality_of(:default_git_depth)
+        .only_integer
+        .is_greater_than_or_equal_to(0)
+        .is_less_than_or_equal_to(1000)
+        .allow_nil
+    end
+  end
+
+  describe '#default_git_depth' do
+    let(:default_value) { described_class::DEFAULT_GIT_DEPTH }
+
+    it 'sets default value for new records' do
+      project = create(:project)
+
+      expect(project.ci_cd_settings.default_git_depth).to eq(default_value)
+    end
+
+    it 'does not set default value if present' do
+      project = build(:project)
+      project.build_ci_cd_settings(default_git_depth: 0)
+      project.save!
+
+      expect(project.reload.ci_cd_settings.default_git_depth).to eq(0)
+    end
+  end
 end

spec/presenters/ci/build_runner_presenter_spec.rb

@@ -119,23 +119,23 @@ describe Ci::BuildRunnerPresenter do
   end
 
   describe '#git_depth' do
-    subject { presenter.git_depth }
-
     let(:build) { create(:ci_build) }
 
-    it 'returns the correct git depth' do
-      is_expected.to eq(0)
-    end
+    subject(:git_depth) { presenter.git_depth }
 
     context 'when GIT_DEPTH variable is specified' do
       before do
         create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
       end
 
-      it 'returns the correct git depth' do
-        is_expected.to eq(1)
+      it 'returns its value' do
+        expect(git_depth).to eq(1)
       end
     end
+
+    it 'defaults to git depth setting for the project' do
+      expect(git_depth).to eq(build.project.default_git_depth)
+    end
   end
 
   describe '#refspecs' do
@@ -143,16 +143,6 @@ describe Ci::BuildRunnerPresenter do
 
     let(:build) { create(:ci_build) }
 
-    it 'returns the correct refspecs' do
-      is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                     '+refs/heads/*:refs/remotes/origin/*')
-    end
-
-    context 'when GIT_DEPTH variable is specified' do
-      before do
-        create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
-      end
-
     it 'returns the correct refspecs' do
       is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
     end
@@ -163,6 +153,16 @@ describe Ci::BuildRunnerPresenter do
       it 'returns the correct refspecs' do
         is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
       end
+
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
+
+        it 'returns the correct refspecs' do
+          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
+                                         '+refs/heads/*:refs/remotes/origin/*')
+        end
       end
     end
 
@@ -173,17 +173,27 @@ describe Ci::BuildRunnerPresenter do
 
       it 'returns the correct refspecs' do
         is_expected
-          .to contain_exactly('+refs/heads/*:refs/remotes/origin/*',
-                              '+refs/tags/*:refs/tags/*',
-                              '+refs/merge-requests/1/head:refs/merge-requests/1/head')
+          .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head')
+      end
+
+      context 'when GIT_DEPTH is zero' do
+        before do
+          create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
+        end
+
+        it 'returns the correct refspecs' do
+          is_expected
+            .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head',
+                                '+refs/heads/*:refs/remotes/origin/*',
+                                '+refs/tags/*:refs/tags/*')
+        end
       end
 
       context 'when pipeline is legacy detached merge request pipeline' do
         let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) }
 
         it 'returns the correct refspecs' do
-          is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
-                                         '+refs/heads/*:refs/remotes/origin/*')
+          is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
         end
       end
     end

spec/requests/api/runner_spec.rb

@@ -444,8 +444,8 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               'sha' => job.sha,
               'before_sha' => job.before_sha,
               'ref_type' => 'branch',
-              'refspecs' => %w[+refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*],
-              'depth' => 0 }
+              'refspecs' => ["+refs/heads/#{job.ref}:refs/remotes/origin/#{job.ref}"],
+              'depth' => project.default_git_depth }
           end
 
           let(:expected_steps) do
@@ -531,7 +531,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 
@@ -587,7 +591,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
               end
             end
 
-            context 'when GIT_DEPTH is not specified' do
+            context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
+              before do
+                project.update!(default_git_depth: nil)
+              end
+
               it 'specifies refspecs' do
                 request_job
 

spec/services/projects/fork_service_spec.rb

@@ -145,6 +145,30 @@ describe Projects::ForkService do
         end
       end
 
+      context "CI/CD settings" do
+        let(:to_project) { fork_project(@from_project, @to_user) }
+
+        context "when origin has git depth specified" do
+          before do
+            @from_project.update(default_git_depth: 42)
+          end
+
+          it "inherits default_git_depth from the origin project" do
+            expect(to_project.default_git_depth).to eq(42)
+          end
+        end
+
+        context "when origin does not define git depth" do
+          before do
+            @from_project.update!(default_git_depth: nil)
+          end
+
+          it "the fork has git depth set to 0" do
+            expect(to_project.default_git_depth).to eq(0)
+          end
+        end
+      end
+
       context "when project has restricted visibility level" do
         context "and only one visibility level is restricted" do
           before do