Merge branch 'stale-environment-cleanup' into 'master'

Add API endpoint for stale env deletion

See merge request gitlab-org/gitlab!52224

app/models/environment.rb

@@ -96,6 +96,22 @@ class Environment < ApplicationRecord
   end
   scope :for_id, -> (id) { where(id: id) }
 
+  scope :stopped_review_apps, -> (before, limit) do
+    stopped
+      .in_review_folder
+      .where("created_at < ?", before)
+      .order("created_at ASC")
+      .limit(limit)
+  end
+
+  scope :scheduled_for_deletion, -> do
+    where.not(auto_delete_at: nil)
+  end
+
+  scope :not_scheduled_for_deletion, -> do
+    where(auto_delete_at: nil)
+  end
+
   enum tier: {
     production: 0,
     staging: 1,
@@ -147,6 +163,10 @@ class Environment < ApplicationRecord
     self.state_machine.states.map(&:name)
   end
 
+  def self.schedule_to_delete(at_time = 1.week.from_now)
+    update_all(auto_delete_at: at_time)
+  end
+
   class << self
     ##
     # This method returns stop actions (jobs) for multiple environments within one

app/services/environments/schedule_to_delete_review_apps_service.rb

+# frozen_string_literal: true
+
+module Environments
+  class ScheduleToDeleteReviewAppsService < ::BaseService
+    include ::Gitlab::ExclusiveLeaseHelpers
+
+    EXCLUSIVE_LOCK_KEY_BASE = 'environments:delete_review_apps:lock'
+    LOCK_TIMEOUT = 2.minutes
+
+    def execute
+      if validation_error = validate
+        return validation_error
+      end
+
+      mark_deletable_environments
+    end
+
+    private
+
+    def key
+      "#{EXCLUSIVE_LOCK_KEY_BASE}:#{project.id}"
+    end
+
+    def dry_run?
+      return true if params[:dry_run].nil?
+
+      params[:dry_run]
+    end
+
+    def validate
+      return if can?(current_user, :destroy_environment, project)
+
+      Result.new(error_message: "You do not have permission to destroy environments in this project", status: :unauthorized)
+    end
+
+    def mark_deletable_environments
+      in_lock(key, ttl: LOCK_TIMEOUT, retries: 1) do
+        unsafe_mark_deletable_environments
+      end
+
+    rescue FailedToObtainLockError
+      Result.new(error_message: "Another process is already processing a delete request. Please retry later.", status: :conflict)
+    end
+
+    def unsafe_mark_deletable_environments
+      result = Result.new
+      environments = project.environments
+                            .not_scheduled_for_deletion
+                            .stopped_review_apps(params[:before], params[:limit])
+
+      # Check if the actor has write permission to a potentially-protected environment.
+      deletable, failed = *environments.partition { |env| current_user.can?(:destroy_environment, env) }
+
+      if deletable.any? && failed.empty?
+        mark_for_deletion(deletable) unless dry_run?
+        result.set_status(:ok)
+        result.set_scheduled_entries(deletable)
+      else
+        result.set_status(
+          :bad_request,
+          error_message: "Failed to authorize deletions for some or all of the environments. Ask someone with more permissions to delete the environments."
+        )
+
+        result.set_unprocessable_entries(failed)
+      end
+
+      result
+    end
+
+    def mark_for_deletion(deletable_environments)
+      Environment.for_id(deletable_environments).schedule_to_delete
+    end
+
+    class Result
+      attr_accessor :scheduled_entries, :unprocessable_entries, :error_message, :status
+
+      def initialize(scheduled_entries: [], unprocessable_entries: [], error_message: nil, status: nil)
+        self.scheduled_entries = scheduled_entries
+        self.unprocessable_entries = unprocessable_entries
+        self.error_message = error_message
+        self.status = status
+      end
+
+      def success?
+        status == :ok
+      end
+
+      def set_status(status, error_message: nil)
+        self.status = status
+        self.error_message = error_message
+      end
+
+      def set_scheduled_entries(entries)
+        self.scheduled_entries = entries
+      end
+
+      def set_unprocessable_entries(entries)
+        self.unprocessable_entries = entries
+      end
+    end
+  end
+end

changelogs/unreleased/stale-environment-cleanup.yml

+---
+title: Add API endpoint for deleting stale review envs
+merge_request: 52224
+author:
+type: added

lib/api/environments.rb

@@ -75,6 +75,33 @@ module API
         end
       end
 
+      desc "Delete multiple stopped review apps" do
+        detail "Remove multiple stopped review environments older than a specific age"
+        success Entities::Environment
+      end
+      params do
+        optional :before, type: Time, desc: "The timestamp before which environments can be deleted. Defaults to 30 days ago.", default: -> { 30.days.ago }
+        optional :limit, type: Integer, desc: "Maximum number of environments to delete. Defaults to 100.", default: 100, values: 1..1000
+        optional :dry_run, type: Boolean, desc: "If set, perform a dry run where no actual deletions will be performed. Defaults to true.", default: true
+      end
+      delete ":id/environments/review_apps" do
+        authorize! :read_environment, user_project
+
+        result = ::Environments::ScheduleToDeleteReviewAppsService.new(user_project, current_user, params).execute
+
+        response = {
+          scheduled_entries: Entities::Environment.represent(result.scheduled_entries),
+          unprocessable_entries: Entities::Environment.represent(result.unprocessable_entries)
+        }
+
+        if result.success?
+          status result.status
+          present response, current_user: current_user
+        else
+          render_api_error!(response.merge!(message: result.error_message), result.status)
+        end
+      end
+
       desc 'Deletes an existing environment' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Environment

spec/factories/environments.rb

@@ -32,6 +32,8 @@ FactoryBot.define do
     end
 
     trait :with_review_app do |environment|
+      sequence(:name) { |n| "review/#{n}" }
+
       transient do
         ref { 'master' }
       end

spec/models/environment_spec.rb

@@ -84,6 +84,62 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching do
     end
   end
 
+  describe ".stopped_review_apps" do
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:old_stopped_review_env) { create(:environment, :with_review_app, :stopped, created_at: 31.days.ago, project: project) }
+    let_it_be(:new_stopped_review_env) { create(:environment, :with_review_app, :stopped, project: project) }
+    let_it_be(:old_active_review_env) { create(:environment, :with_review_app, :available, created_at: 31.days.ago, project: project) }
+    let_it_be(:old_stopped_other_env) { create(:environment, :stopped, created_at: 31.days.ago, project: project) }
+    let_it_be(:new_stopped_other_env) { create(:environment, :stopped, project: project) }
+    let_it_be(:old_active_other_env) { create(:environment, :available, created_at: 31.days.ago, project: project) }
+
+    let(:before) { 30.days.ago }
+    let(:limit) { 1000 }
+
+    subject { project.environments.stopped_review_apps(before, limit) } # rubocop: disable RSpec/SingleLineHook
+
+    it { is_expected.to contain_exactly(old_stopped_review_env) }
+
+    context "current timestamp" do
+      let(:before) { Time.zone.now }
+
+      it { is_expected.to contain_exactly(old_stopped_review_env, new_stopped_review_env) }
+    end
+  end
+
+  describe "scheduled deletion" do
+    let_it_be(:deletable_environment) { create(:environment, auto_delete_at: Time.zone.now) }
+    let_it_be(:undeletable_environment) { create(:environment, auto_delete_at: nil) }
+
+    describe ".scheduled_for_deletion" do
+      subject { described_class.scheduled_for_deletion }
+
+      it { is_expected.to contain_exactly(deletable_environment) }
+    end
+
+    describe ".not_scheduled_for_deletion" do
+      subject { described_class.not_scheduled_for_deletion }
+
+      it { is_expected.to contain_exactly(undeletable_environment) }
+    end
+
+    describe ".schedule_to_delete" do
+      subject { described_class.for_id(deletable_environment).schedule_to_delete }
+
+      it "schedules the record for deletion" do
+        freeze_time do
+          subject
+
+          deletable_environment.reload
+          undeletable_environment.reload
+
+          expect(deletable_environment.auto_delete_at).to eq(1.week.from_now)
+          expect(undeletable_environment.auto_delete_at).to be_nil
+        end
+      end
+    end
+  end
+
   describe 'state machine' do
     it 'invalidates the cache after a change' do
       expect(environment).to receive(:expire_etag_cache)

spec/requests/api/environments_spec.rb

@@ -265,4 +265,76 @@ RSpec.describe API::Environments do
       end
     end
   end
+
+  describe "DELETE /projects/:id/environments/review_apps" do
+    shared_examples "delete stopped review environments" do
+      around do |example|
+        freeze_time { example.run }
+      end
+
+      it "deletes the old stopped review apps" do
+        old_stopped_review_env = create(:environment, :with_review_app, :stopped, created_at: 31.days.ago, project: project)
+        new_stopped_review_env = create(:environment, :with_review_app, :stopped, project: project)
+        old_active_review_env  = create(:environment, :with_review_app, :available, created_at: 31.days.ago, project: project)
+        old_stopped_other_env  = create(:environment, :stopped, created_at: 31.days.ago, project: project)
+        new_stopped_other_env  = create(:environment, :stopped, project: project)
+        old_active_other_env   = create(:environment, :available, created_at: 31.days.ago, project: project)
+
+        delete api("/projects/#{project.id}/environments/review_apps", current_user), params: { dry_run: false }
+        project.environments.reload
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response["scheduled_entries"].size).to eq(1)
+        expect(json_response["scheduled_entries"].first["id"]).to eq(old_stopped_review_env.id)
+        expect(json_response["unprocessable_entries"].size).to eq(0)
+
+        expect(old_stopped_review_env.reload.auto_delete_at).to eq(1.week.from_now)
+        expect(new_stopped_review_env.reload.auto_delete_at).to be_nil
+        expect(old_active_review_env.reload.auto_delete_at).to be_nil
+        expect(old_stopped_other_env.reload.auto_delete_at).to be_nil
+        expect(new_stopped_other_env.reload.auto_delete_at).to be_nil
+        expect(old_active_other_env.reload.auto_delete_at).to be_nil
+      end
+    end
+
+    context "as a maintainer" do
+      it_behaves_like "delete stopped review environments" do
+        let(:current_user) { user }
+      end
+    end
+
+    context "as a developer" do
+      let(:developer) { create(:user) }
+
+      before do
+        project.add_developer(developer)
+      end
+
+      it_behaves_like "delete stopped review environments" do
+        let(:current_user) { developer }
+      end
+    end
+
+    context "as a reporter" do
+      let(:reporter) { create(:user) }
+
+      before do
+        project.add_reporter(reporter)
+      end
+
+      it "rejects the request" do
+        delete api("/projects/#{project.id}/environments/review_apps", reporter)
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+      end
+    end
+
+    context "as a non member" do
+      it "rejects the request" do
+        delete api("/projects/#{project.id}/environments/review_apps", non_member)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
 end

spec/services/environments/schedule_to_delete_review_apps_service_spec.rb

+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Environments::ScheduleToDeleteReviewAppsService do
+  include ExclusiveLeaseHelpers
+
+  let_it_be(:maintainer) { create(:user) }
+  let_it_be(:developer)  { create(:user) }
+  let_it_be(:reporter)   { create(:user) }
+  let_it_be(:project)    { create(:project, :private, :repository, namespace: maintainer.namespace) }
+
+  let(:service)      { described_class.new(project, current_user, before: 30.days.ago, dry_run: dry_run) }
+  let(:dry_run)      { false }
+  let(:current_user) { maintainer }
+
+  before do
+    project.add_maintainer(maintainer)
+    project.add_developer(developer)
+    project.add_reporter(reporter)
+  end
+
+  describe "#execute" do
+    subject { service.execute }
+
+    shared_examples "can schedule for deletion" do
+      let!(:old_stopped_review_env) { create(:environment, :with_review_app, :stopped, created_at: 31.days.ago, project: project) }
+      let!(:new_stopped_review_env) { create(:environment, :with_review_app, :stopped, project: project) }
+      let!(:old_active_review_env)  { create(:environment, :with_review_app, :available, created_at: 31.days.ago, project: project) }
+      let!(:old_stopped_other_env)  { create(:environment, :stopped, created_at: 31.days.ago, project: project) }
+      let!(:new_stopped_other_env)  { create(:environment, :stopped, project: project) }
+      let!(:old_active_other_env)   { create(:environment, :available, created_at: 31.days.ago, project: project) }
+      let!(:already_deleting_env)   { create(:environment, :with_review_app, :stopped, created_at: 31.days.ago, project: project, auto_delete_at: 1.day.from_now) }
+      let(:already_deleting_time)   { already_deleting_env.reload.auto_delete_at }
+
+      context "live run" do
+        let(:dry_run) { false }
+
+        around do |example|
+          freeze_time { example.run }
+        end
+
+        it "marks the correct environment as scheduled_entries" do
+          expect(subject.success?).to be_truthy
+          expect(subject.scheduled_entries).to contain_exactly(old_stopped_review_env)
+          expect(subject.unprocessable_entries).to be_empty
+
+          old_stopped_review_env.reload
+          new_stopped_review_env.reload
+          old_active_review_env.reload
+          old_stopped_other_env.reload
+          new_stopped_other_env.reload
+          old_active_other_env.reload
+          already_deleting_env.reload
+
+          expect(old_stopped_review_env.auto_delete_at).to eq(1.week.from_now)
+          expect(new_stopped_review_env.auto_delete_at).to be_nil
+          expect(old_active_review_env.auto_delete_at).to be_nil
+          expect(old_stopped_other_env.auto_delete_at).to be_nil
+          expect(new_stopped_other_env.auto_delete_at).to be_nil
+          expect(old_active_other_env.auto_delete_at).to be_nil
+          expect(already_deleting_env.auto_delete_at).to eq(already_deleting_time)
+        end
+      end
+
+      context "dry run" do
+        let(:dry_run) { true }
+
+        it "returns the same but doesn't update the record" do
+          expect(subject.success?).to be_truthy
+          expect(subject.scheduled_entries).to contain_exactly(old_stopped_review_env)
+          expect(subject.unprocessable_entries).to be_empty
+
+          old_stopped_review_env.reload
+          new_stopped_review_env.reload
+          old_active_review_env.reload
+          old_stopped_other_env.reload
+          new_stopped_other_env.reload
+          old_active_other_env.reload
+          already_deleting_env.reload
+
+          expect(old_stopped_review_env.auto_delete_at).to be_nil
+          expect(new_stopped_review_env.auto_delete_at).to be_nil
+          expect(old_active_review_env.auto_delete_at).to be_nil
+          expect(old_stopped_other_env.auto_delete_at).to be_nil
+          expect(new_stopped_other_env.auto_delete_at).to be_nil
+          expect(old_active_other_env.auto_delete_at).to be_nil
+          expect(already_deleting_env.auto_delete_at).to eq(already_deleting_time)
+        end
+      end
+
+      describe "execution in parallel" do
+        before do
+          stub_exclusive_lease_taken(service.send(:key))
+        end
+
+        it "does not execute unsafe_mark_scheduled_entries_environments" do
+          expect(service).not_to receive(:unsafe_mark_scheduled_entries_environments)
+
+          expect(subject.success?).to be_falsey
+          expect(subject.status).to eq(:conflict)
+        end
+      end
+    end
+
+    context "as a maintainer" do
+      let(:current_user) { maintainer }
+
+      it_behaves_like "can schedule for deletion"
+    end
+
+    context "as a developer" do
+      let(:current_user) { developer }
+
+      it_behaves_like "can schedule for deletion"
+    end
+
+    context "as a reporter" do
+      let(:current_user) { reporter }
+
+      it "fails to delete environments" do
+        old_stopped_review_env = create(:environment, :with_review_app, :stopped, created_at: 31.days.ago, project: project)
+
+        expect(subject.success?).to be_falsey
+
+        # Both of these should be empty as we fail before testing them
+        expect(subject.scheduled_entries).to be_empty
+        expect(subject.unprocessable_entries).to be_empty
+
+        old_stopped_review_env.reload
+
+        expect(old_stopped_review_env.auto_delete_at).to be_nil
+      end
+    end
+  end
+end