Commit 0e881b2f authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '325285-rake-pages-deployments' into 'master'

Add rake tasks for Pages deployment migration

See merge request gitlab-org/gitlab!57120
parents 44b37595 25346c92
......@@ -14,6 +14,8 @@ class PagesDeployment < ApplicationRecord
scope :older_than, -> (id) { where('id < ?', id) }
scope :migrated_from_legacy_storage, -> { where(file: MIGRATED_FILE_NAME) }
scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
scope :with_files_stored_remotely, -> { where(file_store: ::ObjectStorage::Store::REMOTE) }
validates :file, presence: true
validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES }
......
---
title: Add rake tasks for Pages deployment migration
merge_request: 57120
author:
type: added
# frozen_string_literal: true
module Gitlab
module Pages
class MigrationHelper
def initialize(logger = nil)
@logger = logger
end
def migrate_to_remote_storage
deployments = ::PagesDeployment.with_files_stored_locally
migrate(deployments, ObjectStorage::Store::REMOTE)
end
def migrate_to_local_storage
deployments = ::PagesDeployment.with_files_stored_remotely
migrate(deployments, ObjectStorage::Store::LOCAL)
end
private
def batch_size
ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
end
def migrate(deployments, store)
deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
deployment.file.migrate!(store)
log_success(deployment, store)
rescue => e
log_error(e, deployment)
end
end
def log_success(deployment, store)
logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
end
def log_error(err, deployment)
logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
end
def storage_label(store)
if store == ObjectStorage::Store::LOCAL
'local'
else
'object'
end
end
end
end
end
......@@ -58,5 +58,33 @@ namespace :gitlab do
ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false')
)
end
namespace :deployments do
task migrate_to_object_storage: :gitlab_environment do
logger = Logger.new(STDOUT)
logger.info('Starting transfer of pages deployments to remote storage')
helper = Gitlab::Pages::MigrationHelper.new(logger)
begin
helper.migrate_to_remote_storage
rescue => e
logger.error(e.message)
end
end
task migrate_to_local: :gitlab_environment do
logger = Logger.new(STDOUT)
logger.info('Starting transfer of Pages deployments to local storage')
helper = Gitlab::Pages::MigrationHelper.new(logger)
begin
helper.migrate_to_local_storage
rescue => e
logger.error(e.message)
end
end
end
end
end
......@@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe PagesDeployment do
let_it_be(:project) { create(:project) }
describe 'associations' do
it { is_expected.to belong_to(:project).required }
it { is_expected.to belong_to(:ci_build).optional }
......@@ -28,7 +30,6 @@ RSpec.describe PagesDeployment do
describe '.migrated_from_legacy_storage' do
it 'only returns migrated deployments' do
project = create(:project)
migrated_deployment = create_migrated_deployment(project)
# create one other deployment
create(:pages_deployment, project: project)
......@@ -37,6 +38,27 @@ RSpec.describe PagesDeployment do
end
end
context 'with deployments stored locally and remotely' do
before do
stub_pages_object_storage(::Pages::DeploymentUploader)
end
let!(:remote_deployment) { create(:pages_deployment, project: project, file_store: ::ObjectStorage::Store::REMOTE) }
let!(:local_deployment) { create(:pages_deployment, project: project, file_store: ::ObjectStorage::Store::LOCAL) }
describe '.with_files_stored_locally' do
it 'only returns deployments with files stored locally' do
expect(described_class.with_files_stored_locally).to contain_exactly(local_deployment)
end
end
describe '.with_files_stored_remotely' do
it 'only returns deployments with files stored remotely' do
expect(described_class.with_files_stored_remotely).to contain_exactly(remote_deployment)
end
end
end
describe '#migrated?' do
it 'returns false for normal deployment' do
deployment = create(:pages_deployment)
......@@ -45,7 +67,6 @@ RSpec.describe PagesDeployment do
end
it 'returns true for migrated deployment' do
project = create(:project)
deployment = create_migrated_deployment(project)
expect(deployment.migrated?).to eq(true)
......@@ -67,7 +88,6 @@ RSpec.describe PagesDeployment do
end
describe 'default for file_store' do
let(:project) { create(:project) }
let(:deployment) do
filepath = Rails.root.join("spec/fixtures/pages.zip")
......
......@@ -96,4 +96,80 @@ RSpec.describe 'gitlab:pages' do
expect(PagesDeployment.find_by_id(migrated_deployment.id)).to be_nil
end
end
describe 'gitlab:pages:deployments:migrate_to_object_storage' do
subject { run_rake_task('gitlab:pages:deployments:migrate_to_object_storage') }
before do
stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
end
let!(:deployment) { create(:pages_deployment, file_store: store) }
let(:object_storage_enabled) { true }
context 'when local storage is used' do
let(:store) { ObjectStorage::Store::LOCAL }
context 'and remote storage is defined' do
it 'migrates file to remote storage' do
subject
expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
end
end
context 'and remote storage is not defined' do
let(:object_storage_enabled) { false }
it 'fails to migrate to remote storage' do
subject
expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
end
end
end
context 'when remote storage is used' do
let(:store) { ObjectStorage::Store::REMOTE }
it 'file stays on remote storage' do
subject
expect(deployment.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
end
end
end
describe 'gitlab:pages:deployments:migrate_to_local' do
subject { run_rake_task('gitlab:pages:deployments:migrate_to_local') }
before do
stub_pages_object_storage(::Pages::DeploymentUploader, enabled: object_storage_enabled)
end
let!(:deployment) { create(:pages_deployment, file_store: store) }
let(:object_storage_enabled) { true }
context 'when remote storage is used' do
let(:store) { ObjectStorage::Store::REMOTE }
context 'and job has remote file store defined' do
it 'migrates file to local storage' do
subject
expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
end
end
end
context 'when local storage is used' do
let(:store) { ObjectStorage::Store::LOCAL }
it 'file stays on local storage' do
subject
expect(deployment.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment