Merge branch 'georgekoltsov/project-migration-lfs-objects-export' into 'master'

Export Project LFS Objects as part of GitLab Migration

See merge request gitlab-org/gitlab!77018

app/models/bulk_imports/file_transfer/project_config.rb

@@ -8,6 +8,8 @@ module BulkImports
         group_members
       ).freeze
 
+      LFS_OBJECTS_RELATION = 'lfs_objects'
+
       def import_export_yaml
         ::Gitlab::ImportExport.config_file
       end
@@ -15,6 +17,10 @@ module BulkImports
       def skipped_relations
         SKIPPED_RELATIONS
       end
+
+      def file_relations
+        [UPLOADS_RELATION, LFS_OBJECTS_RELATION]
+      end
     end
   end
 end

app/services/bulk_imports/file_export_service.rb

@@ -26,8 +26,10 @@ module BulkImports
 
     def export_service
       case relation
-      when FileTransfer::ProjectConfig::UPLOADS_RELATION
+      when FileTransfer::BaseConfig::UPLOADS_RELATION
         UploadsExportService.new(portable, export_path)
+      when FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION
+        LfsObjectsExportService.new(portable, export_path)
       else
         raise BulkImports::Error, 'Unsupported relation export type'
       end

app/services/bulk_imports/lfs_objects_export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class LfsObjectsExportService
+    include Gitlab::ImportExport::CommandLineUtil
+
+    BATCH_SIZE = 100
+
+    def initialize(portable, export_path)
+      @portable = portable
+      @export_path = export_path
+      @lfs_json = {}
+    end
+
+    def execute
+      portable.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| # rubocop: disable CodeReuse/ActiveRecord
+        batch.each do |lfs_object|
+          save_lfs_object(lfs_object)
+        end
+
+        append_lfs_json_for_batch(batch)
+      end
+
+      write_lfs_json
+    end
+
+    private
+
+    attr_reader :portable, :export_path, :lfs_json
+
+    def save_lfs_object(lfs_object)
+      destination_filepath = File.join(export_path, lfs_object.oid)
+
+      if lfs_object.local_store?
+        copy_files(lfs_object.file.path, destination_filepath)
+      else
+        download(lfs_object.file.url, destination_filepath)
+      end
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def append_lfs_json_for_batch(lfs_objects_batch)
+      lfs_objects_projects = LfsObjectsProject
+        .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types')
+        .joins(:lfs_object)
+        .where(project: portable, lfs_object: lfs_objects_batch)
+        .group('lfs_objects.oid')
+
+      lfs_objects_projects.each do |group|
+        oid = group.oid
+
+        lfs_json[oid] ||= []
+        lfs_json[oid] += group.repository_types
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def write_lfs_json
+      filepath = File.join(export_path, "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json")
+
+      File.write(filepath, lfs_json.to_json)
+    end
+  end
+end

spec/models/bulk_imports/file_transfer/project_config_spec.rb

@@ -91,4 +91,10 @@ RSpec.describe BulkImports::FileTransfer::ProjectConfig do
       end
     end
   end
+
+  describe '#file_relations' do
+    it 'returns project file relations' do
+      expect(subject.file_relations).to contain_exactly('uploads', 'lfs_objects')
+    end
+  end
 end

spec/services/bulk_imports/file_export_service_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe BulkImports::FileExportService do
   let_it_be(:project) { create(:project) }
   let_it_be(:export_path) { Dir.mktmpdir }
-  let_it_be(:relation) { 'uploads' }
+  let_it_be(:relation) { BulkImports::FileTransfer::BaseConfig::UPLOADS_RELATION }
 
   subject(:service) { described_class.new(project, export_path, relation) }
 
@@ -20,6 +20,20 @@ RSpec.describe BulkImports::FileExportService do
       subject.execute
     end
 
+    context 'when relation is lfs objects' do
+      let_it_be(:relation) { BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION }
+
+      it 'executes lfs objects export service' do
+        expect_next_instance_of(BulkImports::LfsObjectsExportService) do |service|
+          expect(service).to receive(:execute)
+        end
+
+        expect(subject).to receive(:tar_cf).with(archive: File.join(export_path, 'lfs_objects.tar'), dir: export_path)
+
+        subject.execute
+      end
+    end
+
     context 'when unsupported relation is passed' do
       it 'raises an error' do
         service = described_class.new(project, export_path, 'unsupported')

spec/services/bulk_imports/lfs_objects_export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::LfsObjectsExportService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:lfs_json_filename) { "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json" }
+  let_it_be(:remote_url) { 'http://my-object-storage.local' }
+
+  let(:export_path) { Dir.mktmpdir }
+  let(:lfs_object) { create(:lfs_object, :with_file) }
+
+  subject(:service) { described_class.new(project, export_path) }
+
+  before do
+    stub_lfs_object_storage
+
+    %w(wiki design).each do |repository_type|
+      create(
+        :lfs_objects_project,
+        project: project,
+        repository_type: repository_type,
+        lfs_object: lfs_object
+      )
+    end
+
+    project.lfs_objects << lfs_object
+  end
+
+  after do
+    FileUtils.remove_entry(export_path) if Dir.exist?(export_path)
+  end
+
+  describe '#execute' do
+    it 'exports lfs objects and their repository types' do
+      filepath = File.join(export_path, lfs_json_filename)
+
+      service.execute
+
+      expect(File).to exist(File.join(export_path, lfs_object.oid))
+      expect(File).to exist(filepath)
+
+      lfs_json = Gitlab::Json.parse(File.read(filepath))
+
+      expect(lfs_json).to eq(
+        {
+          lfs_object.oid => [
+            LfsObjectsProject.repository_types['wiki'],
+            LfsObjectsProject.repository_types['design'],
+            nil
+          ]
+        }
+      )
+    end
+
+    context 'when lfs object is remotely stored' do
+      let(:lfs_object) { create(:lfs_object, :object_storage) }
+
+      it 'downloads lfs object from object storage' do
+        expect_next_instance_of(LfsObjectUploader) do |instance|
+          expect(instance).to receive(:url).and_return(remote_url)
+        end
+
+        expect(subject).to receive(:download).with(remote_url, File.join(export_path, lfs_object.oid))
+
+        service.execute
+      end
+    end
+  end
+end