Merge branch '325285-rake-pages-deployments' into 'master'

Add rake tasks for Pages deployment migration

See merge request gitlab-org/gitlab!57120

app/models/pages_deployment.rb

@@ -14,6 +14,8 @@ class PagesDeployment < ApplicationRecord
 
   scope :older_than, -> (id) { where('id < ?', id) }
   scope :migrated_from_legacy_storage, -> { where(file: MIGRATED_FILE_NAME) }
+  scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
+  scope :with_files_stored_remotely, -> { where(file_store: ::ObjectStorage::Store::REMOTE) }
 
   validates :file, presence: true
   validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES }

changelogs/unreleased/325285-rake-pages-deployments.yml

+---
+title: Add rake tasks for Pages deployment migration
+merge_request: 57120
+author:
+type: added

lib/gitlab/pages/migration_helper.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Pages
+    class MigrationHelper
+      def initialize(logger = nil)
+        @logger = logger
+      end
+
+      def migrate_to_remote_storage
+        deployments = ::PagesDeployment.with_files_stored_locally
+        migrate(deployments, ObjectStorage::Store::REMOTE)
+      end
+
+      def migrate_to_local_storage
+        deployments = ::PagesDeployment.with_files_stored_remotely
+        migrate(deployments, ObjectStorage::Store::LOCAL)
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+      end
+
+      def migrate(deployments, store)
+        deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
+          deployment.file.migrate!(store)
+
+          log_success(deployment, store)
+        rescue => e
+          log_error(e, deployment)
+        end
+      end
+
+      def log_success(deployment, store)
+        logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
+      end
+
+      def log_error(err, deployment)
+        logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
+      end
+
+      def storage_label(store)
+        if store == ObjectStorage::Store::LOCAL
+          'local'
+        else
+          'object'
+        end
+      end
+    end
+  end
+end

lib/tasks/gitlab/pages.rake

@@ -58,5 +58,33 @@ namespace :gitlab do
         ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false')
       )
     end
+
+    namespace :deployments do
+      task migrate_to_object_storage: :gitlab_environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Starting transfer of pages deployments to remote storage')
+
+        helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+        begin
+          helper.migrate_to_remote_storage
+        rescue => e
+          logger.error(e.message)
+        end
+      end
+
+      task migrate_to_local: :gitlab_environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Starting transfer of Pages deployments to local storage')
+
+        helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+        begin
+          helper.migrate_to_local_storage
+        rescue => e
+          logger.error(e.message)
+        end
+      end
+    end
   end
 end

spec/models/pages_deployment_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe PagesDeployment do
+  let_it_be(:project) { create(:project) }
+
   describe 'associations' do
     it { is_expected.to belong_to(:project).required }
     it { is_expected.to belong_to(:ci_build).optional }
@@ -28,7 +30,6 @@ RSpec.describe PagesDeployment do
 
   describe '.migrated_from_legacy_storage' do
     it 'only returns migrated deployments' do
-      project = create(:project)
       migrated_deployment = create_migrated_deployment(project)
       # create one other deployment
       create(:pages_deployment, project: project)
@@ -37,6 +38,27 @@ RSpec.describe PagesDeployment do
     end
   end
 
+  context 'with deployments stored locally and remotely' do
+    before do
+      stub_pages_object_storage(::Pages::DeploymentUploader)
+    end
+
+    let!(:remote_deployment) { create(:pages_deployment, project: project, file_store: ::ObjectStorage::Store::REMOTE) }
+    let!(:local_deployment) { create(:pages_deployment, project: project, file_store: ::ObjectStorage::Store::LOCAL) }
+
+    describe '.with_files_stored_locally' do
+      it 'only returns deployments with files stored locally' do
+        expect(described_class.with_files_stored_locally).to contain_exactly(local_deployment)
+      end
+    end
+
+    describe '.with_files_stored_remotely' do
+      it 'only returns deployments with files stored remotely' do
+        expect(described_class.with_files_stored_remotely).to contain_exactly(remote_deployment)
+      end
+    end
+  end
+
   describe '#migrated?' do
     it 'returns false for normal deployment' do
       deployment = create(:pages_deployment)
@@ -45,7 +67,6 @@ RSpec.describe PagesDeployment do
     end
 
     it 'returns true for migrated deployment' do
-      project = create(:project)
       deployment = create_migrated_deployment(project)
 
       expect(deployment.migrated?).to eq(true)
@@ -67,7 +88,6 @@ RSpec.describe PagesDeployment do
   end
 
   describe 'default for file_store' do
-    let(:project) { create(:project) }
     let(:deployment) do
       filepath = Rails.root.join("spec/fixtures/pages.zip")
 

spec/tasks/gitlab/pages_rake_spec.rb

@@ -96,4 +96,80 @@ RSpec.describe 'gitlab:pages' do
       expect(PagesDeployment.find_by_id(migrated_deployment.id)).to be_nil
     end
   end
+
+  describe 'gitlab:pages:deployments:migrate_to_object_storage' do
+    subject { run_rake_task('gitlab:pages:deployments:migrate_to_object_storage') }
+
+    before do
+      stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
+    end
+
+    let!(:deployment) { create(:pages_deployment, file_store: store) }
+    let(:object_storage_enabled) { true }
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      context 'and remote storage is defined' do
+        it 'migrates file to remote storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is not defined' do
+        let(:object_storage_enabled) { false }
+
+        it 'fails to migrate to remote storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when remote storage is used' do
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      it 'file stays on remote storage' do
+        subject
+
+        expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
+  end
+
+  describe 'gitlab:pages:deployments:migrate_to_local' do
+    subject { run_rake_task('gitlab:pages:deployments:migrate_to_local') }
+
+    before do
+      stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
+    end
+
+    let!(:deployment) { create(:pages_deployment, file_store: store) }
+    let(:object_storage_enabled) { true }
+
+    context 'when remote storage is used' do
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      context 'and job has remote file store defined' do
+        it 'migrates file to local storage' do
+          subject
+
+          expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      it 'file stays on local storage' do
+        subject
+
+        expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+      end
+    end
+  end
 end