Merge branch '215005-decrease-setup-test-env-duration-from-9-74-minutes' into 'master'

Add timing information and optimize the `setup-test-env` job

See merge request gitlab-org/gitlab!30368

.gitlab/ci/global.gitlab-ci.yml

@@ -46,8 +46,6 @@
     - name: redis:alpine
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
-  cache:
-    key: "debian-stretch-ruby-2.6.6-pg11-node-12.x"
 
 .use-pg11-ee:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
@@ -58,8 +56,6 @@
     - name: elasticsearch:6.4.2
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
-  cache:
-    key: "debian-stretch-ruby-2.6.6-pg11-node-12.x"
 
 # Pin kaniko to v0.16.0 due to https://github.com/GoogleContainerTools/kaniko/issues/1162
 .use-kaniko:

.gitlab/ci/rails.gitlab-ci.yml

 .rails:needs:setup-and-assets:
   needs: ["setup-test-env", "compile-assets pull-cache"]
 
+.rails-cache:
+  cache:
+    key: "ruby-go-cache-v1"
+    paths:
+      - vendor/ruby
+      - vendor/gitaly-ruby
+      - .go/pkg/mod
+    policy: pull
+
 .rails-job-base:
   extends:
     - .default-retry
-    - .default-cache
     - .default-before_script
+    - .rails-cache
 
 #######################################################
 # EE/FOSS: default refs (MRs, master, schedules) jobs #
@@ -13,15 +22,25 @@
   extends:
     - .rails-job-base
   stage: prepare
+  variables:
+    GITLAB_TEST_EAGER_LOAD: "0"
   script:
-    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
-    - scripts/gitaly-test-build  # Do not use 'bundle exec' here
+    - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
+    - run_timed_command "scripts/gitaly-test-build"  # Do not use 'bundle exec' here
   artifacts:
     expire_in: 7d
     paths:
-      - tmp/tests
       - config/secrets.yml
-      - vendor/gitaly-ruby
+      - tmp/tests/gitaly
+      - tmp/tests/gitlab-elasticsearch-indexer
+      - tmp/tests/gitlab-shell
+      - tmp/tests/gitlab-test-fork
+      - tmp/tests/gitlab-test-fork_bare
+      - tmp/tests/gitlab-test
+      - tmp/tests/gitlab-workhorse
+      - tmp/tests/repositories
+      - tmp/tests/second_storage
+    when: always
   cache:
     policy: pull-push
 
@@ -52,8 +71,8 @@ static-analysis:
 downtime_check:
   extends:
     - .rails-job-base
-    - .rails:needs:setup-and-assets
     - .rails:rules:downtime_check
+  needs: ["setup-test-env"]
   stage: test
   variables:
     SETUP_DB: "false"
@@ -176,7 +195,7 @@ gitlab:setup:
     # db/fixtures/development/04_project.rb thanks to SIZE=1 below
     - git clone https://gitlab.com/gitlab-org/gitlab-test.git
        /home/git/repositories/gitlab-org/gitlab-test.git
-    - scripts/gitaly-test-spawn
+    - run_timed_command "scripts/gitaly-test-spawn"
     - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
   artifacts:
     when: on_failure

config/environments/test.rb

 require 'gitlab/testing/request_blocker_middleware'
 require 'gitlab/testing/request_inspector_middleware'
 require 'gitlab/testing/clear_process_memory_cache_middleware'
+require 'gitlab/utils'
 
 Rails.application.configure do
   # Make sure the middleware is inserted first in middleware chain
@@ -43,7 +44,7 @@ Rails.application.configure do
   # Print deprecation notices to the stderr
   config.active_support.deprecation = :stderr
 
-  config.eager_load = true
+  config.eager_load = Gitlab::Utils.to_boolean(ENV['GITLAB_TEST_EAGER_LOAD'], default: true)
 
   config.cache_store = :null_store
 

ee/lib/tasks/gitlab/indexer.rake

@@ -13,7 +13,7 @@ Usage: rake "gitlab:indexer:install[/installation/dir,repo]")
 
       abort "Couldn't find a 'make' binary" unless make
 
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
       Dir.chdir(args.dir) { run_command!([make, 'build']) }
     end

lib/gitlab/task_helpers.rb

@@ -157,8 +157,8 @@ module Gitlab
       Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
     end
 
-    def checkout_or_clone_version(version:, repo:, target_dir:)
-      clone_repo(repo, target_dir) unless Dir.exist?(target_dir)
+    def checkout_or_clone_version(version:, repo:, target_dir:, clone_opts: [])
+      clone_repo(repo, target_dir, clone_opts: clone_opts) unless Dir.exist?(target_dir)
       checkout_version(get_version(version), target_dir)
     end
 
@@ -171,8 +171,8 @@ module Gitlab
       "v#{component_version}"
     end
 
-    def clone_repo(repo, target_dir)
-      run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
+    def clone_repo(repo, target_dir, clone_opts: [])
+      run_command!(%W[#{Gitlab.config.git.bin_path} clone] + clone_opts + %W[-- #{repo} #{target_dir}])
     end
 
     def checkout_version(version, target_dir)

lib/gitlab/utils.rb

@@ -75,12 +75,12 @@ module Gitlab
       str.gsub(/\r?\n/, '')
     end
 
-    def to_boolean(value)
+    def to_boolean(value, default: nil)
       return value if [true, false].include?(value)
       return true if value =~ /^(true|t|yes|y|1|on)$/i
       return false if value =~ /^(false|f|no|n|0|off)$/i
 
-      nil
+      default
     end
 
     def boolean_to_yes_no(bool)

lib/tasks/gitlab/gitaly.rake

@@ -13,7 +13,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
 
       version = Gitlab::GitalyClient.expected_server_version
 
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
       command = []
       _, status = Gitlab::Popen.popen(%w[which gmake])

lib/tasks/gitlab/shell.rake

@@ -12,7 +12,7 @@ namespace :gitlab do
       gitlab_url += '/' unless gitlab_url.end_with?('/')
       target_dir = Gitlab.config.gitlab_shell.path
 
-      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
+      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir, clone_opts: %w[--depth 1])
 
       # Make sure we're on the right tag
       Dir.chdir(target_dir) do

lib/tasks/gitlab/workhorse.rake

@@ -12,7 +12,7 @@ namespace :gitlab do
 
       version = Gitlab::Workhorse.version
 
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
       _, status = Gitlab::Popen.popen(%w[which gmake])
       command = status.zero? ? 'gmake' : 'make'

scripts/gitaly_test.rb

@@ -20,7 +20,7 @@ module GitalyTest
       'HOME' => File.expand_path('tmp/tests'),
       'GEM_PATH' => Gem.path.join(':'),
       'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'),
-      'BUNDLE_FLAGS' => "--jobs=4 --retry=3",
+      'BUNDLE_FLAGS' => "--jobs=4 --retry=3 --quiet",
       'BUNDLE_INSTALL_FLAGS' => nil,
       'BUNDLE_GEMFILE' => gemfile,
       'RUBYOPT' => nil,

scripts/prepare_build.sh

@@ -6,12 +6,17 @@ export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor
 
 if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
   bundle --version
-  bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check
+  run_timed_command "bundle install --clean ${BUNDLE_INSTALL_FLAGS}"
+  run_timed_command "bundle check"
+  # When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env`
+  # job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions).
+  # Uncomment the following line if multiple versions of PG are tested in the same pipeline.
+  # run_timed_command "bundle pristine pg"
 fi
 
 # Only install knapsack after bundle install! Otherwise oddly some native
 # gems could not be found under some circumstance. No idea why, hours wasted.
-retry gem install knapsack --no-document
+run_timed_command "gem install knapsack --no-document"
 
 cp config/gitlab.yml.example config/gitlab.yml
 sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml

scripts/utils.sh

@@ -18,11 +18,8 @@ function setup_db_user_only() {
 }
 
 function setup_db() {
-    setup_db_user_only
-
-    bundle exec rake db:drop db:create db:structure:load db:migrate
-
-    bundle exec rake gitlab:db:setup_ee
+    run_timed_command "setup_db_user_only"
+    run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee"
 }
 
 function install_api_client_dependencies_with_apk() {
@@ -38,6 +35,24 @@ function install_gitlab_gem() {
   gem install gitlab --no-document --version 4.13.0
 }
 
+function run_timed_command() {
+  local cmd="${1}"
+  local start=$(date +%s)
+  echosuccess "\$ ${cmd}"
+  eval "${cmd}"
+  local ret=$?
+  local end=$(date +%s)
+  local runtime=$((end-start))
+
+  if [[ $ret -eq 0 ]]; then
+    echosuccess "==> '${cmd}' succeeded in ${runtime} seconds."
+    return 0
+  else
+    echoerr "==> '${cmd}' failed (${ret}) in ${runtime} seconds."
+    return $ret
+  fi
+}
+
 function echoerr() {
   local header="${2}"
 
@@ -58,6 +73,16 @@ function echoinfo() {
   fi
 }
 
+function echosuccess() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;32m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;32m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
 function get_job_id() {
   local job_name="${1}"
   local query_string="${2:+&${2}}"

spec/lib/gitlab/utils_spec.rb

@@ -130,7 +130,7 @@ describe Gitlab::Utils do
       expect(to_boolean(false)).to be(false)
     end
 
-    it 'converts a valid string to a boolean' do
+    it 'converts a valid value to a boolean' do
       expect(to_boolean(true)).to be(true)
       expect(to_boolean('true')).to be(true)
       expect(to_boolean('YeS')).to be(true)
@@ -146,12 +146,35 @@ describe Gitlab::Utils do
       expect(to_boolean('oFF')).to be(false)
     end
 
-    it 'converts an invalid string to nil' do
+    it 'converts an invalid value to nil' do
       expect(to_boolean('fals')).to be_nil
       expect(to_boolean('yeah')).to be_nil
       expect(to_boolean('')).to be_nil
       expect(to_boolean(nil)).to be_nil
     end
+
+    it 'accepts a default value, and does not return it when a valid value is given' do
+      expect(to_boolean(true, default: false)).to be(true)
+      expect(to_boolean('true', default: false)).to be(true)
+      expect(to_boolean('YeS', default: false)).to be(true)
+      expect(to_boolean('t', default: false)).to be(true)
+      expect(to_boolean('1', default: 'any value')).to be(true)
+      expect(to_boolean('ON', default: 42)).to be(true)
+
+      expect(to_boolean('FaLse', default: true)).to be(false)
+      expect(to_boolean('F', default: true)).to be(false)
+      expect(to_boolean('NO', default: true)).to be(false)
+      expect(to_boolean('n', default: true)).to be(false)
+      expect(to_boolean('0', default: 'any value')).to be(false)
+      expect(to_boolean('oFF', default: 42)).to be(false)
+    end
+
+    it 'accepts a default value, and returns it when an invalid value is given' do
+      expect(to_boolean('fals', default: true)).to eq(true)
+      expect(to_boolean('yeah', default: false)).to eq(false)
+      expect(to_boolean('', default: 'any value')).to eq('any value')
+      expect(to_boolean(nil, default: 42)).to eq(42)
+    end
   end
 
   describe '.boolean_to_yes_no' do

spec/spec_helper.rb

@@ -8,10 +8,12 @@ ENV["IN_MEMORY_APPLICATION_SETTINGS"] = 'true'
 ENV["RSPEC_ALLOW_INVALID_URLS"] = 'true'
 
 require File.expand_path('../config/environment', __dir__)
+
+require 'rspec/mocks'
 require 'rspec/rails'
-require 'shoulda/matchers'
 require 'rspec/retry'
 require 'rspec-parameterized'
+require 'shoulda/matchers'
 require 'test_prof/recipes/rspec/let_it_be'
 
 rspec_profiling_is_configured =

spec/support/helpers/test_env.rb

 # frozen_string_literal: true
 
-require 'rspec/mocks'
-
 module TestEnv
   extend ActiveSupport::Concern
   extend self
@@ -284,29 +282,33 @@ module TestEnv
   end
 
   def setup_factory_repo
-    setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name,
-               BRANCH_SHA)
+    setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name, BRANCH_SHA)
   end
 
   # This repo has a submodule commit that is not present in the main test
   # repository.
   def setup_forked_repo
-    setup_repo(forked_repo_path, forked_repo_path_bare, forked_repo_name,
-               FORKED_BRANCH_SHA)
+    setup_repo(forked_repo_path, forked_repo_path_bare, forked_repo_name, FORKED_BRANCH_SHA)
   end
 
   def setup_repo(repo_path, repo_path_bare, repo_name, refs)
     clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
 
     unless File.directory?(repo_path)
-      system(*%W(#{Gitlab.config.git.bin_path} clone -q #{clone_url} #{repo_path}))
+      puts "\n==> Setting up #{repo_name} repository in #{repo_path}..."
+      start = Time.now
+      system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path}))
+      puts "    #{repo_path} set up in #{Time.now - start} seconds...\n"
     end
 
     set_repo_refs(repo_path, refs)
 
     unless File.directory?(repo_path_bare)
+      puts "\n==> Setting up #{repo_name} bare repository in #{repo_path_bare}..."
+      start = Time.now
       # We must copy bare repositories because we will push to them.
-      system(git_env, *%W(#{Gitlab.config.git.bin_path} clone -q --bare #{repo_path} #{repo_path_bare}))
+      system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --quiet --bare -- #{repo_path} #{repo_path_bare}))
+      puts "    #{repo_path_bare} set up in #{Time.now - start} seconds...\n"
     end
   end
 

spec/tasks/gitlab/gitaly_rake_spec.rb

@@ -46,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do
 
       it 'calls checkout_or_clone_version with the right arguments' do
         expect(main_object)
-          .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
+          .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path, clone_opts: %w[--depth 1])
 
         subject
       end

spec/tasks/gitlab/task_helpers_spec.rb

@@ -28,7 +28,7 @@ describe Gitlab::TaskHelpers do
 
     context "target_dir doesn't exist" do
       it 'clones the repo' do
-        expect(subject).to receive(:clone_repo).with(repo, clone_path)
+        expect(subject).to receive(:clone_repo).with(repo, clone_path, clone_opts: [])
 
         subject.checkout_or_clone_version(version: version, repo: repo, target_dir: clone_path)
       end
@@ -45,6 +45,12 @@ describe Gitlab::TaskHelpers do
         subject.checkout_or_clone_version(version: version, repo: repo, target_dir: clone_path)
       end
     end
+
+    it 'accepts clone_opts' do
+      expect(subject).to receive(:clone_repo).with(repo, clone_path, clone_opts: %w[--depth 1])
+
+      subject.checkout_or_clone_version(version: version, repo: repo, target_dir: clone_path, clone_opts: %w[--depth 1])
+    end
   end
 
   describe '#clone_repo' do
@@ -54,6 +60,13 @@ describe Gitlab::TaskHelpers do
 
       subject.clone_repo(repo, clone_path)
     end
+
+    it 'accepts clone_opts' do
+      expect(subject)
+        .to receive(:run_command!).with(%W[#{Gitlab.config.git.bin_path} clone --depth 1 -- #{repo} #{clone_path}])
+
+      subject.clone_repo(repo, clone_path, clone_opts: %w[--depth 1])
+    end
   end
 
   describe '#checkout_version' do

spec/tasks/gitlab/workhorse_rake_spec.rb

@@ -36,7 +36,7 @@ describe 'gitlab:workhorse namespace rake task' do
 
       it 'calls checkout_or_clone_version with the right arguments' do
         expect(main_object)
-          .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
+          .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path, clone_opts: %w[--depth 1])
 
         run_rake_task('gitlab:workhorse:install', clone_path)
       end