Merge branch 'revert-acd3831e' into 'master'

Revert "Merge branch '118623-remove-db-dependency-from-setup-test-env-job-rc' into 'master'"

See merge request gitlab-org/gitlab!63367

.gitlab/ci/rails.gitlab-ci.yml

@@ -149,16 +149,13 @@ setup-test-env:
     - .rails-job-base
     - .setup-test-env-cache
     - .rails:rules:code-backstage-qa
+    - .use-pg12
   stage: prepare
   variables:
     GITLAB_TEST_EAGER_LOAD: "0"
-    SETUP_DB: "false"
   script:
-    - run_timed_command "scripts/setup-test-env"
-    - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries"
+    - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
     - run_timed_command "scripts/gitaly-test-build"  # Do not use 'bundle exec' here
-    - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K"
-
   artifacts:
     expire_in: 7d
     paths:

ee/lib/tasks/gitlab/indexer.rake

@@ -3,7 +3,7 @@
 namespace :gitlab do
   namespace :indexer do
     desc "GitLab | Indexer | Install or upgrade gitlab-elasticsearch-indexer"
-    task :install, [:dir, :repo] => :with_gitlab_helpers do |t, args|
+    task :install, [:dir, :repo] => :gitlab_environment do |t, args|
       unless args.dir.present?
         abort %(Please specify the directory where you want to install the indexer
 Usage: rake "gitlab:indexer:install[/installation/dir,repo]")

ee/spec/support/helpers/ee/test_env.rb

@@ -2,28 +2,23 @@
 
 module EE
   module TestEnv
-    extend ::Gitlab::Utils::Override
-
-    override :setup_methods
-    def setup_methods
-      (super + [:setup_indexer]).freeze
-    end
-
-    override :post_init
-    def post_init
+    def init(*args, &blk)
       super
 
-      Settings.elasticsearch['indexer_path'] = indexer_bin_path
+      setup_indexer
     end
 
     def setup_indexer
+      indexer_args = [indexer_path, indexer_url].compact.join(',')
+
       component_timed_setup(
         'GitLab Elasticsearch Indexer',
         install_dir: indexer_path,
         version: indexer_version,
-        task: "gitlab:indexer:install",
-        task_args: [indexer_path, indexer_url].compact
+        task: "gitlab:indexer:install[#{indexer_args}]"
       )
+
+      Settings.elasticsearch['indexer_path'] = indexer_bin_path
     end
 
     def indexer_path

lib/gitlab/gitaly_client.rb

@@ -120,7 +120,7 @@ module Gitlab
         raise "storage #{storage.inspect} is missing a gitaly_address"
       end
 
-      unless %w(tcp unix tls).include?(URI(address).scheme)
+      unless URI(address).scheme.in?(%w(tcp unix tls))
         raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
       end
 

lib/gitlab/gitaly_client/storage_settings.rb

@@ -52,7 +52,7 @@ module Gitlab
         @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path']
 
         storage['path'] = Deprecated
-        @hash = ActiveSupport::HashWithIndifferentAccess.new(storage)
+        @hash = storage.with_indifferent_access
       end
 
       def gitaly_address

lib/tasks/gitlab/gitaly.rake

@@ -3,7 +3,7 @@
 namespace :gitlab do
   namespace :gitaly do
     desc 'GitLab | Gitaly | Install or upgrade gitaly'
-    task :install, [:dir, :storage_path, :repo] => :with_gitlab_helpers do |t, args|
+    task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
 
       unless args.dir.present? && args.storage_path.present?

lib/tasks/gitlab/helpers.rake

@@ -6,7 +6,3 @@ StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
 task gitlab_environment: :environment do
   extend SystemCheck::Helpers
 end
-
-task :with_gitlab_helpers do
-  extend SystemCheck::Helpers
-end

scripts/setup-test-env

-#!/usr/bin/env ruby
-
-# frozen_string_literal: true
-
-require 'bundler/setup'
-
-require 'request_store'
-require 'rake'
-require 'active_support/dependencies'
-require 'active_support/dependencies/autoload'
-require 'active_support/core_ext/numeric'
-require 'active_support/string_inquirer'
-
-module Rails
-  extend self
-
-  def root
-    Pathname.new(File.expand_path('..', __dir__))
-  end
-
-  def env
-    @_env ||= ActiveSupport::StringInquirer.new(ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "test")
-  end
-end
-
-ActiveSupport::Dependencies.autoload_paths << 'lib'
-
-load File.expand_path('../lib/tasks/gitlab/helpers.rake', __dir__)
-load File.expand_path('../lib/tasks/gitlab/gitaly.rake', __dir__)
-
-# Required for config/0_inject_enterprise_edition_module.rb, lib/gitlab/access.rb
-require_dependency File.expand_path('../lib/gitlab', __dir__)
-
-require_dependency File.expand_path('../config/initializers/0_inject_enterprise_edition_module', __dir__)
-
-# Require for lib/gitlab/gitaly_client/storage_settings.rb and config/initializers/1_settings.rb
-require 'active_support/hash_with_indifferent_access'
-
-# Required for lib/gitlab/visibility_level.rb and lib/gitlab/safe_request_store.rb
-require 'active_support/concern'
-require 'active_support/core_ext/module/delegation'
-
-# Required for lib/system_check/helpers.rb
-require_dependency File.expand_path('../lib/gitlab/task_helpers', __dir__)
-
-# Required for lib/tasks/gitlab/helpers.rake
-require_dependency File.expand_path('../lib/system_check/helpers', __dir__)
-
-# Required for config/initializers/1_settings.rb
-require 'omniauth'
-require 'omniauth-github'
-require 'etc'
-require_dependency File.expand_path('../lib/gitlab/access', __dir__)
-
-require_dependency File.expand_path('../config/initializers/1_settings', __dir__)
-
-Gitlab.ee do
-  load File.expand_path('../ee/lib/tasks/gitlab/indexer.rake', __dir__)
-
-  require_dependency File.expand_path('../ee/lib/gitlab/elastic/indexer', __dir__)
-  require_dependency File.expand_path('../lib/gitlab/utils/override', __dir__)
-end
-
-require_dependency File.expand_path('../spec/support/helpers/test_env', __dir__)
-
-TestEnv.init

spec/spec_helper.rb

@@ -230,10 +230,6 @@ RSpec.configure do |config|
     Gitlab::Database.set_open_transactions_baseline
   end
 
-  config.append_before do
-    Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
-  end
-
   config.append_after do
     Gitlab::Database.reset_open_transactions_baseline
   end

spec/support/helpers/test_env.rb

 # frozen_string_literal: true
 
-require 'parallel'
-
 module TestEnv
+  extend ActiveSupport::Concern
   extend self
 
   ComponentFailedToInstallError = Class.new(StandardError)
@@ -95,40 +94,50 @@ module TestEnv
   TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
   REPOS_STORAGE = 'default'
   SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
-  SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
-
-  # Can be overriden
-  def setup_methods
-    SETUP_METHODS
-  end
 
   # Test environment
   #
   # See gitlab.yml.example test section for paths
   #
-  def init
+  def init(opts = {})
     unless Rails.env.test?
       puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n"
       exit 1
     end
 
-    start = Time.now
     # Disable mailer for spinach tests
+    disable_mailer if opts[:mailer] == false
+
     clean_test_path
 
-    # Install components in parallel as most of the setup is I/O.
-    Parallel.each(setup_methods) do |method|
-      public_send(method)
-    end
+    setup_gitlab_shell
+
+    setup_gitaly
+
+    # Feature specs are run through Workhorse
+    setup_workhorse
 
-    post_init
+    # Create repository for FactoryBot.create(:project)
+    setup_factory_repo
 
-    puts "\nTest environment set up in #{Time.now - start} seconds"
+    # Create repository for FactoryBot.create(:forked_project_with_submodules)
+    setup_forked_repo
   end
 
-  # Can be overriden
-  def post_init
-    start_gitaly(gitaly_dir)
+  included do |config|
+    config.append_before do
+      set_current_example_group
+    end
+  end
+
+  def disable_mailer
+    allow_any_instance_of(NotificationService).to receive(:mailer)
+      .and_return(double.as_null_object)
+  end
+
+  def enable_mailer
+    allow_any_instance_of(NotificationService).to receive(:mailer)
+      .and_call_original
   end
 
   # Clean /tmp/tests
@@ -155,11 +164,12 @@ module TestEnv
   end
 
   def setup_gitaly
+    install_gitaly_args = [gitaly_dir, repos_path, gitaly_url].compact.join(',')
+
     component_timed_setup('Gitaly',
       install_dir: gitaly_dir,
       version: Gitlab::GitalyClient.expected_server_version,
-      task: "gitlab:gitaly:install",
-      task_args: [gitaly_dir, repos_path, gitaly_url].compact) do
+      task: "gitlab:gitaly:install[#{install_gitaly_args}]") do
         Gitlab::SetupHelper::Gitaly.create_configuration(
           gitaly_dir,
           { 'default' => repos_path },
@@ -180,6 +190,8 @@ module TestEnv
         )
         Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
       end
+
+    start_gitaly(gitaly_dir)
   end
 
   def gitaly_socket_path
@@ -261,18 +273,19 @@ module TestEnv
     raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds"
   end
 
-  # Feature specs are run through Workhorse
   def setup_workhorse
     start = Time.now
     return if skip_compile_workhorse?
 
+    puts "\n==> Setting up GitLab Workhorse..."
+
     FileUtils.rm_rf(workhorse_dir)
     Gitlab::SetupHelper::Workhorse.compile_into(workhorse_dir)
     Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil)
 
     File.write(workhorse_tree_file, workhorse_tree) if workhorse_source_clean?
 
-    puts "==> GitLab Workhorse set up in #{Time.now - start} seconds...\n"
+    puts "    GitLab Workhorse set up in #{Time.now - start} seconds...\n"
   end
 
   def skip_compile_workhorse?
@@ -336,12 +349,10 @@ module TestEnv
     ENV.fetch('GITLAB_WORKHORSE_URL', nil)
   end
 
-  # Create repository for FactoryBot.create(:project)
   def setup_factory_repo
     setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name, BRANCH_SHA)
   end
 
-  # Create repository for FactoryBot.create(:forked_project_with_submodules)
   # This repo has a submodule commit that is not present in the main test
   # repository.
   def setup_forked_repo
@@ -352,18 +363,20 @@ module TestEnv
     clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git"
 
     unless File.directory?(repo_path)
+      puts "\n==> Setting up #{repo_name} repository in #{repo_path}..."
       start = Time.now
       system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path}))
-      puts "==> #{repo_path} set up in #{Time.now - start} seconds...\n"
+      puts "    #{repo_path} set up in #{Time.now - start} seconds...\n"
     end
 
     set_repo_refs(repo_path, refs)
 
     unless File.directory?(repo_path_bare)
+      puts "\n==> Setting up #{repo_name} bare repository in #{repo_path_bare}..."
       start = Time.now
       # We must copy bare repositories because we will push to them.
       system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --quiet --bare -- #{repo_path} #{repo_path_bare}))
-      puts "==> #{repo_path_bare} set up in #{Time.now - start} seconds...\n"
+      puts "    #{repo_path_bare} set up in #{Time.now - start} seconds...\n"
     end
   end
 
@@ -455,6 +468,10 @@ module TestEnv
 
   private
 
+  def set_current_example_group
+    Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
+  end
+
   # These are directories that should be preserved at cleanup time
   def test_dirs
     @test_dirs ||= %w[
@@ -509,7 +526,7 @@ module TestEnv
     end
   end
 
-  def component_timed_setup(component, install_dir:, version:, task:, task_args: [])
+  def component_timed_setup(component, install_dir:, version:, task:)
     start = Time.now
 
     ensure_component_dir_name_is_correct!(component, install_dir)
@@ -518,16 +535,17 @@ module TestEnv
     return if File.exist?(install_dir) && ci?
 
     if component_needs_update?(install_dir, version)
+      puts "\n==> Setting up #{component}..."
       # Cleanup the component entirely to ensure we start fresh
       FileUtils.rm_rf(install_dir)
 
-      unless Rake::Task[task].invoke(*task_args)
+      unless system('rake', task)
         raise ComponentFailedToInstallError
       end
 
       yield if block_given?
 
-      puts "==> #{component} set up in #{Time.now - start} seconds...\n"
+      puts "    #{component} set up in #{Time.now - start} seconds...\n"
     end
   rescue ComponentFailedToInstallError
     puts "\n#{component} failed to install, cleaning up #{install_dir}!\n"