Commit a5fb3ebd authored by Pavel Shutsin's avatar Pavel Shutsin

Merge branch 'georgekoltsov/project-migration-lfs-objects-export' into 'master'

Export Project LFS Objects as part of GitLab Migration

See merge request gitlab-org/gitlab!77018
parents bbb4a2ae 7867db2c
......@@ -8,6 +8,8 @@ module BulkImports
group_members
).freeze
LFS_OBJECTS_RELATION = 'lfs_objects'
def import_export_yaml
::Gitlab::ImportExport.config_file
end
......@@ -15,6 +17,10 @@ module BulkImports
def skipped_relations
SKIPPED_RELATIONS
end
def file_relations
[UPLOADS_RELATION, LFS_OBJECTS_RELATION]
end
end
end
end
......@@ -26,8 +26,10 @@ module BulkImports
def export_service
case relation
when FileTransfer::ProjectConfig::UPLOADS_RELATION
when FileTransfer::BaseConfig::UPLOADS_RELATION
UploadsExportService.new(portable, export_path)
when FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION
LfsObjectsExportService.new(portable, export_path)
else
raise BulkImports::Error, 'Unsupported relation export type'
end
......
# frozen_string_literal: true
module BulkImports
class LfsObjectsExportService
include Gitlab::ImportExport::CommandLineUtil
BATCH_SIZE = 100
def initialize(portable, export_path)
@portable = portable
@export_path = export_path
@lfs_json = {}
end
def execute
portable.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| # rubocop: disable CodeReuse/ActiveRecord
batch.each do |lfs_object|
save_lfs_object(lfs_object)
end
append_lfs_json_for_batch(batch)
end
write_lfs_json
end
private
attr_reader :portable, :export_path, :lfs_json
def save_lfs_object(lfs_object)
destination_filepath = File.join(export_path, lfs_object.oid)
if lfs_object.local_store?
copy_files(lfs_object.file.path, destination_filepath)
else
download(lfs_object.file.url, destination_filepath)
end
end
# rubocop: disable CodeReuse/ActiveRecord
def append_lfs_json_for_batch(lfs_objects_batch)
lfs_objects_projects = LfsObjectsProject
.select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types')
.joins(:lfs_object)
.where(project: portable, lfs_object: lfs_objects_batch)
.group('lfs_objects.oid')
lfs_objects_projects.each do |group|
oid = group.oid
lfs_json[oid] ||= []
lfs_json[oid] += group.repository_types
end
end
# rubocop: enable CodeReuse/ActiveRecord
def write_lfs_json
filepath = File.join(export_path, "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json")
File.write(filepath, lfs_json.to_json)
end
end
end
......@@ -91,4 +91,10 @@ RSpec.describe BulkImports::FileTransfer::ProjectConfig do
end
end
end
describe '#file_relations' do
it 'returns project file relations' do
expect(subject.file_relations).to contain_exactly('uploads', 'lfs_objects')
end
end
end
......@@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe BulkImports::FileExportService do
let_it_be(:project) { create(:project) }
let_it_be(:export_path) { Dir.mktmpdir }
let_it_be(:relation) { 'uploads' }
let_it_be(:relation) { BulkImports::FileTransfer::BaseConfig::UPLOADS_RELATION }
subject(:service) { described_class.new(project, export_path, relation) }
......@@ -20,6 +20,20 @@ RSpec.describe BulkImports::FileExportService do
subject.execute
end
context 'when relation is lfs objects' do
let_it_be(:relation) { BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION }
it 'executes lfs objects export service' do
expect_next_instance_of(BulkImports::LfsObjectsExportService) do |service|
expect(service).to receive(:execute)
end
expect(subject).to receive(:tar_cf).with(archive: File.join(export_path, 'lfs_objects.tar'), dir: export_path)
subject.execute
end
end
context 'when unsupported relation is passed' do
it 'raises an error' do
service = described_class.new(project, export_path, 'unsupported')
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::LfsObjectsExportService do
let_it_be(:project) { create(:project) }
let_it_be(:lfs_json_filename) { "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json" }
let_it_be(:remote_url) { 'http://my-object-storage.local' }
let(:export_path) { Dir.mktmpdir }
let(:lfs_object) { create(:lfs_object, :with_file) }
subject(:service) { described_class.new(project, export_path) }
before do
stub_lfs_object_storage
%w(wiki design).each do |repository_type|
create(
:lfs_objects_project,
project: project,
repository_type: repository_type,
lfs_object: lfs_object
)
end
project.lfs_objects << lfs_object
end
after do
FileUtils.remove_entry(export_path) if Dir.exist?(export_path)
end
describe '#execute' do
it 'exports lfs objects and their repository types' do
filepath = File.join(export_path, lfs_json_filename)
service.execute
expect(File).to exist(File.join(export_path, lfs_object.oid))
expect(File).to exist(filepath)
lfs_json = Gitlab::Json.parse(File.read(filepath))
expect(lfs_json).to eq(
{
lfs_object.oid => [
LfsObjectsProject.repository_types['wiki'],
LfsObjectsProject.repository_types['design'],
nil
]
}
)
end
context 'when lfs object is remotely stored' do
let(:lfs_object) { create(:lfs_object, :object_storage) }
it 'downloads lfs object from object storage' do
expect_next_instance_of(LfsObjectUploader) do |instance|
expect(instance).to receive(:url).and_return(remote_url)
end
expect(subject).to receive(:download).with(remote_url, File.join(export_path, lfs_object.oid))
service.execute
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment