Merge branch 'georgekoltsov/project-uploads-relation-export' into 'master'

Add project uploads export to Relations Export API

See merge request gitlab-org/gitlab!71586

app/models/bulk_imports/export.rb

@@ -53,7 +53,7 @@ module BulkImports
     end
 
     def relation_definition
-      config.portable_tree[:include].find { |include| include[relation.to_sym] }
+      config.relation_definition_for(relation)
     end
 
     def config

app/models/bulk_imports/file_transfer/base_config.rb

@@ -22,15 +22,25 @@ module BulkImports
       end
 
       def export_path
-        strong_memoize(:export_path) do
-          relative_path = File.join(base_export_path, SecureRandom.hex)
+        @export_path ||= Dir.mktmpdir('bulk_imports')
+      end
 
-          ::Gitlab::ImportExport.export_path(relative_path: relative_path)
+      def portable_relations
+        tree_relations + file_relations - skipped_relations
       end
+
+      def tree_relation?(relation)
+        tree_relations.include?(relation)
       end
 
-      def portable_relations
-        import_export_config.dig(:tree, portable_class_sym).keys.map(&:to_s) - skipped_relations
+      def file_relation?(relation)
+        file_relations.include?(relation)
+      end
+
+      def tree_relation_definition_for(relation)
+        return unless tree_relation?(relation)
+
+        portable_tree[:include].find { |include| include[relation.to_sym] }
       end
 
       private
@@ -44,7 +54,7 @@ module BulkImports
       end
 
       def import_export_config
-        ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h
+        @config ||= ::Gitlab::ImportExport::Config.new(config: import_export_yaml).to_h
       end
 
       def portable_class
@@ -63,8 +73,12 @@ module BulkImports
         raise NotImplementedError
       end
 
-      def base_export_path
-        raise NotImplementedError
+      def tree_relations
+        import_export_config.dig(:tree, portable_class_sym).keys.map(&:to_s)
+      end
+
+      def file_relations
+        []
       end
 
       def skipped_relations

app/models/bulk_imports/file_transfer/group_config.rb

@@ -3,16 +3,14 @@
 module BulkImports
   module FileTransfer
     class GroupConfig < BaseConfig
-      def base_export_path
-        portable.full_path
-      end
+      SKIPPED_RELATIONS = %w(members).freeze
 
       def import_export_yaml
         ::Gitlab::ImportExport.group_config_file
       end
 
       def skipped_relations
-        @skipped_relations ||= %w(members)
+        SKIPPED_RELATIONS
       end
     end
   end

app/models/bulk_imports/file_transfer/project_config.rb

@@ -3,16 +3,23 @@
 module BulkImports
   module FileTransfer
     class ProjectConfig < BaseConfig
-      def base_export_path
-        portable.disk_path
-      end
+      UPLOADS_RELATION = 'uploads'
+
+      SKIPPED_RELATIONS = %w(
+        project_members
+        group_members
+      ).freeze
 
       def import_export_yaml
         ::Gitlab::ImportExport.config_file
       end
 
+      def file_relations
+        [UPLOADS_RELATION]
+      end
+
       def skipped_relations
-        @skipped_relations ||= %w(project_members group_members)
+        SKIPPED_RELATIONS
       end
     end
   end

app/services/bulk_imports/file_export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class FileExportService
+    include Gitlab::ImportExport::CommandLineUtil
+
+    def initialize(portable, export_path, relation)
+      @portable = portable
+      @export_path = export_path
+      @relation = relation
+    end
+
+    def execute
+      export_service.execute
+
+      archive_exported_data
+    end
+
+    def exported_filename
+      "#{relation}.tar"
+    end
+
+    private
+
+    attr_reader :export_path, :portable, :relation
+
+    def export_service
+      case relation
+      when FileTransfer::ProjectConfig::UPLOADS_RELATION
+        UploadsExportService.new(portable, export_path)
+      else
+        raise BulkImports::Error, 'Unsupported relation export type'
+      end
+    end
+
+    def archive_exported_data
+      archive_file = File.join(export_path, exported_filename)
+
+      tar_cf(archive: archive_file, dir: export_path)
+    end
+  end
+end

app/services/bulk_imports/relation_export_service.rb

@@ -9,20 +9,23 @@ module BulkImports
       @portable = portable
       @relation = relation
       @jid = jid
+      @config = FileTransfer.config_for(portable)
     end
 
     def execute
       find_or_create_export! do |export|
         remove_existing_export_file!(export)
-        serialize_relation_to_file(export.relation_definition)
+        export_service.execute
         compress_exported_relation
         upload_compressed_file(export)
       end
+    ensure
+      FileUtils.remove_entry(config.export_path)
     end
 
     private
 
-    attr_reader :user, :portable, :relation, :jid
+    attr_reader :user, :portable, :relation, :jid, :config
 
     def find_or_create_export!
       validate_user_permissions!
@@ -55,52 +58,28 @@ module BulkImports
       upload.save!
     end
 
-    def serialize_relation_to_file(relation_definition)
-      serializer.serialize_relation(relation_definition)
+    def export_service
+      @export_service ||= if config.tree_relation?(relation)
+                            TreeExportService.new(portable, config.export_path, relation)
+                          elsif config.file_relation?(relation)
+                            FileExportService.new(portable, config.export_path, relation)
+                          else
+                            raise BulkImports::Error, 'Unsupported export relation'
                           end
-
-    def compress_exported_relation
-      gzip(dir: export_path, filename: ndjson_filename)
     end
 
     def upload_compressed_file(export)
-      compressed_filename = File.join(export_path, "#{ndjson_filename}.gz")
+      compressed_file = File.join(config.export_path, "#{export_service.exported_filename}.gz")
+
       upload = ExportUpload.find_or_initialize_by(export_id: export.id) # rubocop: disable CodeReuse/ActiveRecord
 
-      File.open(compressed_filename) { |file| upload.export_file = file }
+      File.open(compressed_file) { |file| upload.export_file = file }
 
       upload.save!
     end
 
-    def config
-      @config ||= FileTransfer.config_for(portable)
-    end
-
-    def export_path
-      @export_path ||= config.export_path
-    end
-
-    def portable_tree
-      @portable_tree ||= config.portable_tree
-    end
-
-    # rubocop: disable CodeReuse/Serializer
-    def serializer
-      @serializer ||= ::Gitlab::ImportExport::Json::StreamingSerializer.new(
-        portable,
-        portable_tree,
-        json_writer,
-        exportable_path: ''
-      )
-    end
-    # rubocop: enable CodeReuse/Serializer
-
-    def json_writer
-      @json_writer ||= ::Gitlab::ImportExport::Json::NdjsonWriter.new(export_path)
-    end
-
-    def ndjson_filename
-      @ndjson_filename ||= "#{relation}.ndjson"
+    def compress_exported_relation
+      gzip(dir: config.export_path, filename: export_service.exported_filename)
     end
   end
 end

app/services/bulk_imports/tree_export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class TreeExportService
+    def initialize(portable, export_path, relation)
+      @portable = portable
+      @export_path = export_path
+      @relation = relation
+      @config = FileTransfer.config_for(portable)
+    end
+
+    def execute
+      relation_definition = config.tree_relation_definition_for(relation)
+
+      raise BulkImports::Error, 'Unsupported relation export type' unless relation_definition
+
+      serializer.serialize_relation(relation_definition)
+    end
+
+    def exported_filename
+      "#{relation}.ndjson"
+    end
+
+    private
+
+    attr_reader :export_path, :portable, :relation, :config
+
+    # rubocop: disable CodeReuse/Serializer
+    def serializer
+      ::Gitlab::ImportExport::Json::StreamingSerializer.new(
+        portable,
+        config.portable_tree,
+        json_writer,
+        exportable_path: ''
+      )
+    end
+    # rubocop: enable CodeReuse/Serializer
+
+    def json_writer
+      ::Gitlab::ImportExport::Json::NdjsonWriter.new(export_path)
+    end
+  end
+end

app/services/bulk_imports/uploads_export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class UploadsExportService
+    include Gitlab::ImportExport::CommandLineUtil
+
+    BATCH_SIZE = 100
+
+    def initialize(portable, export_path)
+      @portable = portable
+      @export_path = export_path
+    end
+
+    def execute
+      portable.uploads.find_each(batch_size: BATCH_SIZE) do |upload| # rubocop: disable CodeReuse/ActiveRecord
+        uploader = upload.retrieve_uploader
+
+        next unless upload.exist?
+        next unless uploader.file
+
+        subdir_path = export_subdir_path(upload)
+        mkdir_p(subdir_path)
+        download_or_copy_upload(uploader, File.join(subdir_path, uploader.filename))
+      rescue Errno::ENAMETOOLONG => e
+        # Do not fail entire export process if downloaded file has filename that exceeds 255 characters.
+        # Ignore raised exception, skip such upload, log the error and keep going with the export instead.
+        Gitlab::ErrorTracking.log_exception(e, portable_id: portable.id, portable_class: portable.class.name, upload_id: upload.id)
+      end
+    end
+
+    private
+
+    attr_reader :portable, :export_path
+
+    def export_subdir_path(upload)
+      subdir = if upload.path == avatar_path
+                 'avatar'
+               else
+                 upload.try(:secret).to_s
+               end
+
+      File.join(export_path, subdir)
+    end
+
+    def avatar_path
+      @avatar_path ||= portable.avatar&.upload&.path
+    end
+  end
+end

lib/gitlab/import_export/command_line_util.rb

@@ -14,6 +14,10 @@ module Gitlab
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
+      def tar_cf(archive:, dir:)
+        tar_with_options(archive: archive, dir: dir, options: 'cf')
+      end
+
       def gzip(dir:, filename:)
         gzip_with_options(dir: dir, filename: filename)
       end
@@ -59,19 +63,29 @@ module Gitlab
       end
 
       def tar_with_options(archive:, dir:, options:)
-        execute(%W(tar -#{options} #{archive} -C #{dir} .))
+        execute_cmd(%W(tar -#{options} #{archive} -C #{dir} .))
       end
 
       def untar_with_options(archive:, dir:, options:)
-        execute(%W(tar -#{options} #{archive} -C #{dir}))
-        execute(%W(chmod -R #{UNTAR_MASK} #{dir}))
+        execute_cmd(%W(tar -#{options} #{archive} -C #{dir}))
+        execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir}))
       end
 
-      def execute(cmd)
+      # rubocop:disable Gitlab/ModuleWithInstanceVariables
+      def execute_cmd(cmd)
         output, status = Gitlab::Popen.popen(cmd)
-        @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status == 0 # rubocop:disable Gitlab/ModuleWithInstanceVariables
-        status == 0
+
+        return true if status == 0
+
+        if @shared.respond_to?(:error)
+          @shared.error(Gitlab::ImportExport::Error.new(output.to_s))
+
+          false
+        else
+          raise Gitlab::ImportExport::Error, 'System call failed'
+        end
       end
+      # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
       def git_bin_path
         Gitlab.config.git.bin_path

spec/lib/gitlab/import_export/command_line_util_spec.rb

@@ -8,6 +8,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
   let(:path) { "#{Dir.tmpdir}/symlink_test" }
   let(:archive) { 'spec/fixtures/symlink_export.tar.gz' }
   let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
+  let(:tmpdir) { Dir.mktmpdir }
 
   subject do
     Class.new do
@@ -26,6 +27,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
 
   after do
     FileUtils.rm_rf(path)
+    FileUtils.remove_entry(tmpdir)
   end
 
   it 'has the right mask for project.json' do
@@ -55,7 +57,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
 
   describe '#gunzip' do
     it 'decompresses specified file' do
-      tmpdir = Dir.mktmpdir
       filename = 'labels.ndjson.gz'
       gz_filepath = "spec/fixtures/bulk_imports/gz/#{filename}"
       FileUtils.copy_file(gz_filepath, File.join(tmpdir, filename))
@@ -63,8 +64,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
       subject.gunzip(dir: tmpdir, filename: filename)
 
       expect(File.exist?(File.join(tmpdir, 'labels.ndjson'))).to eq(true)
-
-      FileUtils.remove_entry(tmpdir)
     end
 
     context 'when exception occurs' do
@@ -73,4 +72,33 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
       end
     end
   end
+
+  describe '#tar_cf' do
+    let(:archive_dir) { Dir.mktmpdir }
+
+    after do
+      FileUtils.remove_entry(archive_dir)
+    end
+
+    it 'archives a folder without compression' do
+      archive_file = File.join(archive_dir, 'archive.tar')
+
+      result = subject.tar_cf(archive: archive_file, dir: tmpdir)
+
+      expect(result).to eq(true)
+      expect(File.exist?(archive_file)).to eq(true)
+    end
+
+    context 'when something goes wrong' do
+      it 'raises an error' do
+        expect(Gitlab::Popen).to receive(:popen).and_return(['Error', 1])
+
+        klass = Class.new do
+          include Gitlab::ImportExport::CommandLineUtil
+        end.new
+
+        expect { klass.tar_cf(archive: 'test', dir: 'test') }.to raise_error(Gitlab::ImportExport::Error, 'System call failed')
+      end
+    end
+  end
 end

spec/models/bulk_imports/file_transfer/group_config_spec.rb

@@ -23,10 +23,8 @@ RSpec.describe BulkImports::FileTransfer::GroupConfig do
   end
 
   describe '#export_path' do
-    it 'returns correct export path' do
-      expect(::Gitlab::ImportExport).to receive(:storage_path).and_return('storage_path')
-
-      expect(subject.export_path).to eq("storage_path/#{exportable.full_path}/#{hex}")
+    it 'returns tmpdir location' do
+      expect(subject.export_path).to include(File.join(Dir.tmpdir, 'bulk_imports'))
     end
   end
 

spec/models/bulk_imports/file_transfer/project_config_spec.rb

@@ -23,10 +23,8 @@ RSpec.describe BulkImports::FileTransfer::ProjectConfig do
   end
 
   describe '#export_path' do
-    it 'returns correct export path' do
-      expect(::Gitlab::ImportExport).to receive(:storage_path).and_return('storage_path')
-
-      expect(subject.export_path).to eq("storage_path/#{exportable.disk_path}/#{hex}")
+    it 'returns tmpdir location' do
+      expect(subject.export_path).to include(File.join(Dir.tmpdir, 'bulk_imports'))
     end
   end
 
@@ -51,4 +49,46 @@ RSpec.describe BulkImports::FileTransfer::ProjectConfig do
       expect(subject.relation_excluded_keys('project')).to include('creator_id')
     end
   end
+
+  describe '#tree_relation?' do
+    context 'when it is a tree relation' do
+      it 'returns true' do
+        expect(subject.tree_relation?('labels')).to eq(true)
+      end
+    end
+
+    context 'when it is not a tree relation' do
+      it 'returns false' do
+        expect(subject.tree_relation?('example')).to eq(false)
+      end
+    end
+  end
+
+  describe '#file_relation?' do
+    context 'when it is a file relation' do
+      it 'returns true' do
+        expect(subject.file_relation?('uploads')).to eq(true)
+      end
+    end
+
+    context 'when it is not a file relation' do
+      it 'returns false' do
+        expect(subject.file_relation?('example')).to eq(false)
+      end
+    end
+  end
+
+  describe '#tree_relation_definition_for' do
+    it 'returns relation definition' do
+      expected = { service_desk_setting: { except: [:outgoing_name, :file_template_project_id], include: [] } }
+
+      expect(subject.tree_relation_definition_for('service_desk_setting')).to eq(expected)
+    end
+
+    context 'when relation is not tree relation' do
+      it 'returns' do
+        expect(subject.tree_relation_definition_for('example')).to be_nil
+      end
+    end
+  end
 end

spec/services/bulk_imports/file_export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::FileExportService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:export_path) { Dir.mktmpdir }
+  let_it_be(:relation) { 'uploads' }
+
+  subject(:service) { described_class.new(project, export_path, relation) }
+
+  describe '#execute' do
+    it 'executes export service and archives exported data' do
+      expect_next_instance_of(BulkImports::UploadsExportService) do |service|
+        expect(service).to receive(:execute)
+      end
+
+      expect(subject).to receive(:tar_cf).with(archive: File.join(export_path, 'uploads.tar'), dir: export_path)
+
+      subject.execute
+    end
+
+    context 'when unsupported relation is passed' do
+      it 'raises an error' do
+        service = described_class.new(project, export_path, 'unsupported')
+
+        expect { service.execute }.to raise_error(BulkImports::Error, 'Unsupported relation export type')
+      end
+    end
+  end
+
+  describe '#exported_filename' do
+    it 'returns filename of the exported file' do
+      expect(subject.exported_filename).to eq('uploads.tar')
+    end
+  end
+end

spec/services/bulk_imports/relation_export_service_spec.rb

@@ -7,12 +7,14 @@ RSpec.describe BulkImports::RelationExportService do
   let_it_be(:relation) { 'labels' }
   let_it_be(:user) { create(:user) }
   let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project) }
   let_it_be(:label) { create(:group_label, group: group) }
   let_it_be(:export_path) { "#{Dir.tmpdir}/relation_export_service_spec/tree" }
   let_it_be_with_reload(:export) { create(:bulk_import_export, group: group, relation: relation) }
 
   before do
     group.add_owner(user)
+    project.add_maintainer(user)
 
     allow(export).to receive(:export_path).and_return(export_path)
   end
@@ -25,6 +27,10 @@ RSpec.describe BulkImports::RelationExportService do
 
   describe '#execute' do
     it 'exports specified relation and marks export as finished' do
+      expect_next_instance_of(BulkImports::TreeExportService) do |service|
+        expect(service).to receive(:execute).and_call_original
+      end
+
       subject.execute
 
       expect(export.reload.upload.export_file).to be_present
@@ -43,6 +49,18 @@ RSpec.describe BulkImports::RelationExportService do
       expect(export.upload.export_file).to be_present
     end
 
+    context 'when exporting a file relation' do
+      it 'uses file export service' do
+        service = described_class.new(user, project, 'uploads', jid)
+
+        expect_next_instance_of(BulkImports::FileExportService) do |service|
+          expect(service).to receive(:execute)
+        end
+
+        service.execute
+      end
+    end
+
     context 'when export record does not exist' do
       let(:another_group) { create(:group) }
 

spec/services/bulk_imports/tree_export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::TreeExportService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:export_path) { Dir.mktmpdir }
+  let_it_be(:relation) { 'issues' }
+
+  subject(:service) { described_class.new(project, export_path, relation) }
+
+  describe '#execute' do
+    it 'executes export service and archives exported data' do
+      expect_next_instance_of(Gitlab::ImportExport::Json::StreamingSerializer) do |serializer|
+        expect(serializer).to receive(:serialize_relation)
+      end
+
+      subject.execute
+    end
+
+    context 'when unsupported relation is passed' do
+      it 'raises an error' do
+        service = described_class.new(project, export_path, 'unsupported')
+
+        expect { service.execute }.to raise_error(BulkImports::Error, 'Unsupported relation export type')
+      end
+    end
+  end
+
+  describe '#exported_filename' do
+    it 'returns filename of the exported file' do
+      expect(subject.exported_filename).to eq('issues.ndjson')
+    end
+  end
+end