Terminate connection before copying from main

Template databases must have no other sessions while it is being copied.
See https://www.postgresql.org/docs/current/manage-ag-templatedbs.html

lib/tasks/dev.rake

@@ -27,6 +27,32 @@ namespace :dev do
     Rails.application.eager_load!
   end
 
+  # If there are any clients connected to the DB, PostgreSQL won't let
+  # you drop the database. It's possible that Sidekiq, Puma, or
+  # some other client will be hanging onto a connection, preventing
+  # the DROP DATABASE from working. To workaround this problem, this
+  # method terminates all the connections so that a subsequent DROP
+  # will work.
+  desc "Used to drop all connections in development"
+  task :terminate_all_connections do
+    # In production, we might want to prevent ourselves from shooting
+    # ourselves in the foot, so let's only do this in a test or
+    # development environment.
+    unless Rails.env.production?
+      cmd = <<~SQL
+      SELECT pg_terminate_backend(pg_stat_activity.pid)
+      FROM pg_stat_activity
+      WHERE datname = current_database()
+        AND pid <> pg_backend_pid();
+      SQL
+
+      Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection|
+        connection.execute(cmd)
+      rescue ActiveRecord::NoDatabaseError
+      end
+    end
+  end
+
   databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
 
   namespace :copy_db do
@@ -37,6 +63,8 @@ namespace :dev do
 
       desc "Copies the #{name} database from the main database"
       task name => :environment do
+        Rake::Task["dev:terminate_all_connections"].invoke
+
         db_config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: name)
 
         ApplicationRecord.connection.create_database(db_config.database, template: ApplicationRecord.connection_db_config.database)

lib/tasks/gitlab/setup.rake

@@ -30,7 +30,7 @@ namespace :gitlab do
     # In production, we might want to prevent ourselves from shooting
     # ourselves in the foot, so let's only do this in a test or
     # development environment.
-    terminate_all_connections unless Rails.env.production?
+    Rake::Task["dev:terminate_all_connections"].invoke unless Rails.env.production?
 
     Rake::Task["db:reset"].invoke
     Rake::Task["db:seed_fu"].invoke
@@ -38,24 +38,4 @@ namespace :gitlab do
     puts "Quitting...".color(:red)
     exit 1
   end
-
-  # If there are any clients connected to the DB, PostgreSQL won't let
-  # you drop the database. It's possible that Sidekiq, Puma, or
-  # some other client will be hanging onto a connection, preventing
-  # the DROP DATABASE from working. To workaround this problem, this
-  # method terminates all the connections so that a subsequent DROP
-  # will work.
-  def self.terminate_all_connections
-    cmd = <<~SQL
-      SELECT pg_terminate_backend(pg_stat_activity.pid)
-      FROM pg_stat_activity
-      WHERE datname = current_database()
-        AND pid <> pg_backend_pid();
-    SQL
-
-    Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection|
-      connection.execute(cmd)
-    rescue ActiveRecord::NoDatabaseError
-    end
-  end
 end

spec/tasks/dev_rake_spec.rb

@@ -40,6 +40,58 @@ RSpec.describe 'dev rake tasks' do
     end
   end
 
+  describe 'terminate_all_connections' do
+    let(:connections) do
+      Gitlab::Database.database_base_models.values.filter_map do |model|
+        model.connection if Gitlab::Database.db_config_share_with(model.connection_db_config).nil?
+      end
+    end
+
+    def expect_connections_to_be_terminated
+      expect(Gitlab::Database::EachDatabase).to receive(:each_database_connection)
+        .with(include_shared: false)
+        .and_call_original
+
+      expect(connections).to all(receive(:execute).with(/SELECT pg_terminate_backend/))
+    end
+
+    def expect_connections_not_to_be_terminated
+      connections.each do |connection|
+        expect(connection).not_to receive(:execute)
+      end
+    end
+
+    subject(:terminate_task) { run_rake_task('dev:terminate_all_connections') }
+
+    it 'terminates all connections' do
+      expect_connections_to_be_terminated
+
+      terminate_task
+    end
+
+    context 'when in the production environment' do
+      it 'does not terminate connections' do
+        expect(Rails.env).to receive(:production?).and_return(true)
+        expect_connections_not_to_be_terminated
+
+        terminate_task
+      end
+    end
+
+    context 'when a database is not found' do
+      before do
+        skip_if_multiple_databases_not_setup
+      end
+
+      it 'continues to next connection' do
+        expect(connections.first).to receive(:execute).and_raise(ActiveRecord::NoDatabaseError)
+        expect(connections.second).to receive(:execute).with(/SELECT pg_terminate_backend/)
+
+        terminate_task
+      end
+    end
+  end
+
   context 'multiple databases' do
     before do
       skip_if_multiple_databases_not_setup
@@ -48,6 +100,8 @@ RSpec.describe 'dev rake tasks' do
     context 'with a valid database' do
       describe 'copy_db:ci' do
         before do
+          allow(Rake::Task['dev:terminate_all_connections']).to receive(:invoke)
+
           configurations = instance_double(ActiveRecord::DatabaseConfigurations)
           allow(ActiveRecord::Base).to receive(:configurations).and_return(configurations)
           allow(configurations).to receive(:configs_for).with(env_name: Rails.env, name: 'ci').and_return(ci_configuration)
@@ -63,6 +117,8 @@ RSpec.describe 'dev rake tasks' do
             template: ApplicationRecord.connection_db_config.database
           )
 
+          expect(Rake::Task['dev:terminate_all_connections']).to receive(:invoke)
+
           run_rake_task('dev:copy_db:ci')
         end
 

spec/tasks/gitlab/setup_rake_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe 'gitlab:setup namespace rake tasks', :silence_stdout do
   before do
     Rake.application.rake_require 'active_record/railties/databases'
     Rake.application.rake_require 'tasks/seed_fu'
+    Rake.application.rake_require 'tasks/dev'
     Rake.application.rake_require 'tasks/gitlab/setup'
   end
 
@@ -22,12 +23,6 @@ RSpec.describe 'gitlab:setup namespace rake tasks', :silence_stdout do
     let(:server_service1) { double(:server_service) }
     let(:server_service2) { double(:server_service) }
 
-    let(:connections) do
-      Gitlab::Database.database_base_models.values.filter_map do |model|
-        model.connection if Gitlab::Database.db_config_share_with(model.connection_db_config).nil?
-      end
-    end
-
     before do
       allow(Gitlab).to receive_message_chain('config.repositories.storages').and_return(storages)
 
@@ -102,18 +97,6 @@ RSpec.describe 'gitlab:setup namespace rake tasks', :silence_stdout do
       end
     end
 
-    context 'when the database is not found when terminating connections' do
-      it 'continues setting up the database', :aggregate_failures do
-        expect_gitaly_connections_to_be_checked
-
-        expect(connections).to all(receive(:execute).and_raise(ActiveRecord::NoDatabaseError))
-
-        expect_database_to_be_setup
-
-        setup_task
-      end
-    end
-
     def expect_gitaly_connections_to_be_checked
       expect(Gitlab::GitalyClient::ServerService).to receive(:new).with('name1').and_return(server_service1)
       expect(server_service1).to receive(:info)
@@ -123,17 +106,11 @@ RSpec.describe 'gitlab:setup namespace rake tasks', :silence_stdout do
     end
 
     def expect_connections_to_be_terminated
-      expect(Gitlab::Database::EachDatabase).to receive(:each_database_connection)
-        .with(include_shared: false)
-        .and_call_original
-
-      expect(connections).to all(receive(:execute).with(/SELECT pg_terminate_backend/))
+      expect(Rake::Task['dev:terminate_all_connections']).to receive(:invoke)
     end
 
     def expect_connections_not_to_be_terminated
-      connections.each do |connection|
-        expect(connection).not_to receive(:execute)
-      end
+      expect(Rake::Task['dev:terminate_all_connections']).not_to receive(:invoke)
     end
 
     def expect_database_to_be_setup