Merge branch 'auto-delete-unusued-environments' into 'master'

Archive old deployments (Delete old deployment refs)

See merge request gitlab-org/gitlab!73628

app/models/deployment.rb

@@ -12,6 +12,8 @@ class Deployment < ApplicationRecord
   StatusUpdateError = Class.new(StandardError)
   StatusSyncError = Class.new(StandardError)
 
+  ARCHIVABLE_OFFSET = 50_000
+
   belongs_to :project, required: true
   belongs_to :environment, required: true
   belongs_to :cluster, class_name: 'Clusters::Cluster', optional: true
@@ -100,6 +102,10 @@ class Deployment < ApplicationRecord
       deployment.run_after_commit do
         Deployments::UpdateEnvironmentWorker.perform_async(id)
         Deployments::LinkMergeRequestWorker.perform_async(id)
+
+        if ::Feature.enabled?(:deployments_archive, deployment.project, default_enabled: :yaml)
+          Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id)
+        end
       end
     end
 
@@ -133,6 +139,14 @@ class Deployment < ApplicationRecord
     skipped: 5
   }
 
+  def self.archivables_in(project, limit:)
+    start_iid = project.deployments.order(iid: :desc).limit(1)
+      .select("(iid - #{ARCHIVABLE_OFFSET}) AS start_iid")
+
+    project.deployments.preload(:environment).where('iid <= (?)', start_iid)
+      .where(archived: false).limit(limit)
+  end
+
   def self.last_for_environment(environment)
     ids = self
       .for_environment(environment)

app/services/deployments/archive_in_project_service.rb

+# frozen_string_literal: true
+
+module Deployments
+  # This service archives old deploymets and deletes deployment refs for
+  # keeping the project repository performant.
+  class ArchiveInProjectService < ::BaseService
+    BATCH_SIZE = 100
+
+    def execute
+      unless ::Feature.enabled?(:deployments_archive, project, default_enabled: :yaml)
+        return error('Feature flag is not enabled')
+      end
+
+      deployments = Deployment.archivables_in(project, limit: BATCH_SIZE)
+
+      return success(result: :empty) if deployments.empty?
+
+      ids = deployments.map(&:id)
+      ref_paths = deployments.map(&:ref_path)
+
+      project.repository.delete_refs(*ref_paths)
+      project.deployments.id_in(ids).update_all(archived: true)
+
+      success(result: :archived, count: ids.count)
+    end
+  end
+end

app/workers/all_queues.yml

@@ -723,6 +723,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: deployment:deployments_archive_in_project
+  :worker_name: Deployments::ArchiveInProjectWorker
+  :feature_category: :continuous_delivery
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 3
+  :idempotent: true
+  :tags: []
 - :name: deployment:deployments_drop_older_deployments
   :worker_name: Deployments::DropOlderDeploymentsWorker
   :feature_category: :continuous_delivery

app/workers/deployments/archive_in_project_worker.rb

+# frozen_string_literal: true
+
+module Deployments
+  class ArchiveInProjectWorker
+    include ApplicationWorker
+
+    queue_namespace :deployment
+    feature_category :continuous_delivery
+    idempotent!
+    deduplicate :until_executed, including_scheduled: true
+    data_consistency :delayed
+
+    def perform(project_id)
+      Project.find_by_id(project_id).try do |project|
+        Deployments::ArchiveInProjectService.new(project, nil).execute
+      end
+    end
+  end
+end

config/feature_flags/development/deployments_archive.yml

+---
+name: deployments_archive
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/345027
+milestone: '14.5'
+type: development
+group: group::release
+default_enabled: false

db/migrate/20211105010101_add_archived_column_to_deployments.rb

+# frozen_string_literal: true
+
+class AddArchivedColumnToDeployments < Gitlab::Database::Migration[1.0]
+  enable_lock_retries!
+
+  def change
+    add_column :deployments, :archived, :boolean, default: false, null: false
+  end
+end

db/migrate/20211110010101_add_index_on_unarchived_deployments.rb

+# frozen_string_literal: true
+
+class AddIndexOnUnarchivedDeployments < Gitlab::Database::Migration[1.0]
+  INDEX_NAME = 'index_deployments_on_archived_project_id_iid'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :deployments, %i[archived project_id iid], name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :deployments, INDEX_NAME
+  end
+end

db/schema_migrations/20211105010101

+26c534cdae8630e3f28ad2b61a1049aaab5c5b7a1b761f0961831b621e148ed3
\ No newline at end of file

db/schema_migrations/20211110010101

+63495b9f9ca2d4fa121b75eea36f2923942a6e11f27bef2c51414e00ccd48973
\ No newline at end of file

db/structure.sql

@@ -13375,7 +13375,8 @@ CREATE TABLE deployments (
     status smallint NOT NULL,
     finished_at timestamp with time zone,
     cluster_id integer,
-    deployable_id bigint
+    deployable_id bigint,
+    archived boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE deployments_id_seq
@@ -25625,6 +25626,8 @@ CREATE UNIQUE INDEX index_deployment_clusters_on_cluster_id_and_deployment_id ON
 
 CREATE INDEX index_deployment_merge_requests_on_merge_request_id ON deployment_merge_requests USING btree (merge_request_id);
 
+CREATE INDEX index_deployments_on_archived_project_id_iid ON deployments USING btree (archived, project_id, iid);
+
 CREATE INDEX index_deployments_on_cluster_id_and_status ON deployments USING btree (cluster_id, status);
 
 CREATE INDEX index_deployments_on_created_at ON deployments USING btree (created_at);

doc/ci/environments/index.md

@@ -714,6 +714,29 @@ fetch line:
 fetch = +refs/environments/*:refs/remotes/origin/environments/*
 ```
 
+### Archive Old Deployments
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73628) in GitLab 14.5.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `deployments_archive`. On GitLab.com, this feature will be rolled out gradually.
+
+When a new deployment happens in your project,
+GitLab creates [a special Git-ref to the deployment](#check-out-deployments-locally).
+Since these Git-refs are populated from the remote GitLab repository,
+you could find that some Git operations, such as `git-fetch` and `git-pull`,
+become slower as the number of deployments in your project increases.
+
+To maintain the efficiency of your Git operations, GitLab keeps
+only recent deployment refs (up to 50,000) and deletes the rest of the old deployment refs.
+Archived deployments are still available, in the UI or by using the API, for auditing purposes.
+Also, you can still fetch the deployed commit from the repository
+with specifying the commit SHA (for example, `git checkout <deployment-sha>`), even after archive.
+
+NOTE:
+GitLab preserves all commits as [`keep-around` refs](../../user/project/repository/reducing_the_repo_size_using_git.md)
+so that deployed commits are not garbage collected, even if it's not referenced by the deployment refs.
+
 ### Scope environments with specs
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2112) in GitLab Premium 9.4.
@@ -911,3 +934,19 @@ To fix this, use one of the following solutions:
     script: deploy review app
     environment: review/$CI_COMMIT_REF_SLUG
   ```
+
+### Deployment refs are not found
+
+Starting from GitLab 14.5, GitLab [deletes old deployment refs](#archive-old-deployments)
+to keep your Git repository performant.
+
+If you have to restore archived Git-refs, please ask an administrator of your self-managed GitLab instance
+to execute the following command on Rails console:
+
+```ruby
+Project.find_by_full_path(<your-project-full-path>).deployments.where(archived: true).each(&:create_ref)
+```
+
+Please note that GitLab could drop this support in the future for the performance concern.
+You can open an issue in [GitLab Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues/new)
+to discuss the behavior of this feature.

spec/models/deployment_spec.rb

@@ -385,6 +385,43 @@ RSpec.describe Deployment do
     end
   end
 
+  describe '.archivables_in' do
+    subject { described_class.archivables_in(project, limit: limit) }
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:deployment_1) { create(:deployment, project: project) }
+    let_it_be(:deployment_2) { create(:deployment, project: project) }
+    let_it_be(:deployment_3) { create(:deployment, project: project) }
+
+    let(:limit) { 100 }
+
+    context 'when there are no archivable deployments in the project' do
+      it 'returns nothing' do
+        expect(subject).to be_empty
+      end
+    end
+
+    context 'when there are archivable deployments in the project' do
+      before do
+        stub_const("::Deployment::ARCHIVABLE_OFFSET", 1)
+      end
+
+      it 'returns all archivable deployments' do
+        expect(subject.count).to eq(2)
+        expect(subject).to contain_exactly(deployment_1, deployment_2)
+      end
+
+      context 'with limit' do
+        let(:limit) { 1 }
+
+        it 'takes the limit into account' do
+          expect(subject.count).to eq(1)
+          expect(subject.take).to be_in([deployment_1, deployment_2])
+        end
+      end
+    end
+  end
+
   describe 'scopes' do
     describe 'last_for_environment' do
       let(:production) { create(:environment) }
@@ -774,6 +811,7 @@ RSpec.describe Deployment do
     it 'schedules workers when finishing a deploy' do
       expect(Deployments::UpdateEnvironmentWorker).to receive(:perform_async)
       expect(Deployments::LinkMergeRequestWorker).to receive(:perform_async)
+      expect(Deployments::ArchiveInProjectWorker).to receive(:perform_async)
       expect(Deployments::HooksWorker).to receive(:perform_async)
 
       expect(deploy.update_status('success')).to eq(true)

spec/services/deployments/archive_in_project_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deployments::ArchiveInProjectService do
+  let_it_be(:project) { create(:project, :repository) }
+
+  let(:service) { described_class.new(project, nil) }
+
+  describe '#execute' do
+    subject { service.execute }
+
+    context 'when there are archivable deployments' do
+      let!(:deployments) { create_list(:deployment, 3, project: project) }
+      let!(:deployment_refs) { deployments.map(&:ref_path) }
+
+      before do
+        deployments.each(&:create_ref)
+        allow(Deployment).to receive(:archivables_in) { deployments }
+      end
+
+      it 'returns result code' do
+        expect(subject[:result]).to eq(:archived)
+        expect(subject[:status]).to eq(:success)
+        expect(subject[:count]).to eq(3)
+      end
+
+      it 'archives the deployment' do
+        expect(deployments.map(&:archived?)).to be_all(false)
+        expect(deployment_refs_exist?).to be_all(true)
+
+        subject
+
+        deployments.each(&:reload)
+        expect(deployments.map(&:archived?)).to be_all(true)
+        expect(deployment_refs_exist?).to be_all(false)
+      end
+
+      context 'when ref does not exist by some reason' do
+        before do
+          project.repository.delete_refs(*deployment_refs)
+        end
+
+        it 'does not raise an error' do
+          expect(deployment_refs_exist?).to be_all(false)
+
+          expect { subject }.not_to raise_error
+
+          expect(deployment_refs_exist?).to be_all(false)
+        end
+      end
+
+      context 'when deployments_archive feature flag is disabled' do
+        before do
+          stub_feature_flags(deployments_archive: false)
+        end
+
+        it 'does not do anything' do
+          expect(subject[:status]).to eq(:error)
+          expect(subject[:message]).to eq('Feature flag is not enabled')
+        end
+      end
+
+      def deployment_refs_exist?
+        deployment_refs.map { |path| project.repository.ref_exists?(path) }
+      end
+    end
+
+    context 'when there are no archivable deployments' do
+      before do
+        allow(Deployment).to receive(:archivables_in) { Deployment.none }
+      end
+
+      it 'returns result code' do
+        expect(subject[:result]).to eq(:empty)
+        expect(subject[:status]).to eq(:success)
+      end
+    end
+  end
+end

spec/workers/deployments/archive_in_project_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deployments::ArchiveInProjectWorker do
+  subject { described_class.new.perform(deployment&.project_id) }
+
+  describe '#perform' do
+    let(:deployment) { create(:deployment, :success) }
+
+    it 'executes Deployments::ArchiveInProjectService' do
+      expect(Deployments::ArchiveInProjectService)
+          .to receive(:new).with(deployment.project, nil).and_call_original
+
+      subject
+    end
+  end
+end