Merge branch 'georgekoltsov/add-group-relations-export-worker-and-api' into 'master'

Add Group relations export API

See merge request gitlab-org/gitlab!59978

app/models/bulk_imports/export.rb

@@ -36,6 +36,15 @@ module BulkImports
       end
     end
 
+    def self.config(exportable)
+      case exportable
+      when ::Project
+        Exports::ProjectConfig.new(exportable)
+      when ::Group
+        Exports::GroupConfig.new(exportable)
+      end
+    end
+
     def exportable_relation?
       return unless exportable
 
@@ -54,12 +63,7 @@ module BulkImports
 
     def config
       strong_memoize(:config) do
-        case exportable
-        when ::Project
-          Exports::ProjectConfig.new(exportable)
-        when ::Group
-          Exports::GroupConfig.new(exportable)
-        end
+        self.class.config(exportable)
       end
     end
   end

app/models/bulk_imports/exports/base_config.rb

@@ -13,11 +13,6 @@ module BulkImports
         attributes_finder.find_root(exportable_class_sym)
       end
 
-      def validate_user_permissions!(user)
-        user.can?(ability, exportable) ||
-          raise(::Gitlab::ImportExport::Error.permission_error(user, exportable))
-      end
-
       def export_path
         strong_memoize(:export_path) do
           relative_path = File.join(base_export_path, SecureRandom.hex)
@@ -56,10 +51,6 @@ module BulkImports
         raise NotImplementedError
       end
 
-      def ability
-        raise NotImplementedError
-      end
-
       def base_export_path
         raise NotImplementedError
       end

app/models/bulk_imports/exports/group_config.rb

@@ -3,8 +3,6 @@
 module BulkImports
   module Exports
     class GroupConfig < BaseConfig
-      private
-
       def base_export_path
         exportable.full_path
       end
@@ -12,10 +10,6 @@ module BulkImports
       def import_export_yaml
         ::Gitlab::ImportExport.group_config_file
       end
-
-      def ability
-        :admin_group
-      end
     end
   end
 end

app/models/bulk_imports/exports/project_config.rb

@@ -3,8 +3,6 @@
 module BulkImports
   module Exports
     class ProjectConfig < BaseConfig
-      private
-
       def base_export_path
         exportable.disk_path
       end
@@ -12,10 +10,6 @@ module BulkImports
       def import_export_yaml
         ::Gitlab::ImportExport.config_file
       end
-
-      def ability
-        :admin_project
-      end
     end
   end
 end

app/models/group.rb

@@ -704,6 +704,10 @@ class Group < Namespace
     Gitlab::ServiceDesk.supported? && all_projects.service_desk_enabled.exists?
   end
 
+  def to_ability_name
+    model_name.singular
+  end
+
   private
 
   def update_two_factor_requirement

app/services/bulk_imports/export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class ExportService
+    def initialize(exportable:, user:)
+      @exportable = exportable
+      @current_user = user
+    end
+
+    def execute
+      Export.config(exportable).exportable_relations.each do |relation|
+        RelationExportWorker.perform_async(current_user.id, exportable.id, exportable.class.name, relation)
+      end
+
+      ServiceResponse.success
+    rescue StandardError => e
+      ServiceResponse.error(
+        message: e.class,
+        http_status: :unprocessable_entity
+      )
+    end
+
+    private
+
+    attr_reader :exportable, :current_user
+  end
+end

app/services/bulk_imports/relation_export_service.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class RelationExportService
+    include Gitlab::ImportExport::CommandLineUtil
+
+    def initialize(user, exportable, relation, jid)
+      @user = user
+      @exportable = exportable
+      @relation = relation
+      @jid = jid
+    end
+
+    def execute
+      find_or_create_export! do |export|
+        remove_existing_export_file!(export)
+        serialize_relation_to_file(export.relation_definition)
+        compress_exported_relation
+        upload_compressed_file(export)
+      end
+    end
+
+    private
+
+    attr_reader :user, :exportable, :relation, :jid
+
+    def find_or_create_export!
+      validate_user_permissions!
+
+      export = exportable.bulk_import_exports.safe_find_or_create_by!(relation: relation)
+      export.update!(status_event: 'start', jid: jid)
+
+      yield export
+
+      export.update!(status_event: 'finish', error: nil)
+    rescue StandardError => e
+      Gitlab::ErrorTracking.track_exception(e, exportable_id: exportable.id, exportable_type: exportable.class.name)
+
+      export&.update(status_event: 'fail_op', error: e.class)
+    end
+
+    def validate_user_permissions!
+      ability = "admin_#{exportable.to_ability_name}"
+
+      user.can?(ability, exportable) ||
+        raise(::Gitlab::ImportExport::Error.permission_error(user, exportable))
+    end
+
+    def remove_existing_export_file!(export)
+      upload = export.upload
+
+      return unless upload&.export_file&.file
+
+      upload.remove_export_file!
+      upload.save!
+    end
+
+    def serialize_relation_to_file(relation_definition)
+      serializer.serialize_relation(relation_definition)
+    end
+
+    def compress_exported_relation
+      gzip(dir: export_path, filename: ndjson_filename)
+    end
+
+    def upload_compressed_file(export)
+      compressed_filename = File.join(export_path, "#{ndjson_filename}.gz")
+      upload = ExportUpload.find_or_initialize_by(export_id: export.id) # rubocop: disable CodeReuse/ActiveRecord
+
+      File.open(compressed_filename) { |file| upload.export_file = file }
+
+      upload.save!
+    end
+
+    def export_config
+      @export_config ||= Export.config(exportable)
+    end
+
+    def export_path
+      @export_path ||= export_config.export_path
+    end
+
+    def exportable_tree
+      @exportable_tree ||= export_config.exportable_tree
+    end
+
+    # rubocop: disable CodeReuse/Serializer
+    def serializer
+      @serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new(
+        exportable,
+        exportable_tree,
+        json_writer,
+        exportable_path: ''
+      )
+    end
+    # rubocop: enable CodeReuse/Serializer
+
+    def json_writer
+      @json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path)
+    end
+
+    def ndjson_filename
+      @ndjson_filename ||= "#{relation}.ndjson"
+    end
+  end
+end

app/workers/all_queues.yml

@@ -1837,6 +1837,16 @@
   :idempotent:
   :tags:
   - :exclude_from_kubernetes
+- :name: bulk_imports_relation_export
+  :worker_name: BulkImports::RelationExportWorker
+  :feature_category: :importers
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags:
+  - :exclude_from_kubernetes
 - :name: chat_notification
   :worker_name: ChatNotificationWorker
   :feature_category: :chatops

app/workers/bulk_imports/relation_export_worker.rb

+# frozen_string_literal: true
+
+module BulkImports
+  class RelationExportWorker
+    include ApplicationWorker
+    include ExceptionBacktrace
+
+    idempotent!
+    loggable_arguments 2, 3
+    feature_category :importers
+    tags :exclude_from_kubernetes
+    sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION
+
+    def perform(user_id, exportable_id, exportable_class, relation)
+      user = User.find(user_id)
+      exportable = exportable(exportable_id, exportable_class)
+
+      RelationExportService.new(user, exportable, relation, jid).execute
+    end
+
+    private
+
+    def exportable(exportable_id, exportable_class)
+      exportable_class.classify.constantize.find(exportable_id)
+    end
+  end
+end

changelogs/unreleased/georgekoltsov-add-group-relations-export-worker-and-api.yml

+---
+title: Add Group relations export API
+merge_request: 59978
+author:
+type: added

config/sidekiq_queues.yml

@@ -58,6 +58,8 @@
   - 1
 - - bulk_imports_pipeline
   - 1
+- - bulk_imports_relation_export
+  - 1
 - - chaos
   - 2
 - - chat_notification

lib/api/entities/bulk_imports/export_status.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    module BulkImports
+      class ExportStatus < Grape::Entity
+        expose :relation
+        expose :status
+        expose :error
+        expose :updated_at
+      end
+    end
+  end
+end

lib/api/group_export.rb

@@ -43,6 +43,43 @@ module API
           render_api_error!(message: 'Group export could not be started.')
         end
       end
+
+      desc 'Start relations export' do
+        detail 'This feature was introduced in GitLab 13.12'
+      end
+      post ':id/export_relations' do
+        response = ::BulkImports::ExportService.new(exportable: user_group, user: current_user).execute
+
+        if response.success?
+          accepted!
+        else
+          render_api_error!(message: 'Group relations export could not be started.')
+        end
+      end
+
+      desc 'Download relations export' do
+        detail 'This feature was introduced in GitLab 13.12'
+      end
+      params do
+        requires :relation, type: String, desc: 'Group relation name'
+      end
+      get ':id/export_relations/download' do
+        export = user_group.bulk_import_exports.find_by_relation(params[:relation])
+        file = export&.upload&.export_file
+
+        if file
+          present_carrierwave_file!(file)
+        else
+          render_api_error!('404 Not found', 404)
+        end
+      end
+
+      desc 'Relations export status' do
+        detail 'This feature was introduced in GitLab 13.12'
+      end
+      get ':id/export_relations/status' do
+        present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus
+      end
     end
   end
 end

lib/gitlab/import_export/command_line_util.rb

@@ -14,6 +14,19 @@ module Gitlab
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
+      def gzip(dir:, filename:)
+        filepath = File.join(dir, filename)
+        cmd = %W(gzip #{filepath})
+
+        _, status = Gitlab::Popen.popen(cmd)
+
+        if status == 0
+          status
+        else
+          raise Gitlab::ImportExport::Error.file_compression_error
+        end
+      end
+
       def mkdir_p(path)
         FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE)
         FileUtils.chmod(DEFAULT_DIR_MODE, path)

lib/gitlab/import_export/json/streaming_serializer.rb

@@ -38,16 +38,6 @@ module Gitlab
           end
         end
 
-        private
-
-        attr_reader :json_writer, :relations_schema, :exportable
-
-        def serialize_root
-          attributes = exportable.as_json(
-            relations_schema.merge(include: nil, preloads: nil))
-          json_writer.write_attributes(@exportable_path, attributes)
-        end
-
         def serialize_relation(definition)
           raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
           raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
@@ -64,6 +54,16 @@ module Gitlab
           end
         end
 
+        private
+
+        attr_reader :json_writer, :relations_schema, :exportable
+
+        def serialize_root
+          attributes = exportable.as_json(
+            relations_schema.merge(include: nil, preloads: nil))
+          json_writer.write_attributes(@exportable_path, attributes)
+        end
+
         def serialize_many_relations(key, records, options)
           enumerator = Enumerator.new do |items|
             key_preloads = preloads&.dig(key)

spec/lib/api/entities/bulk_imports/export_status_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Entities::BulkImports::ExportStatus do
+  let_it_be(:export) { create(:bulk_import_export) }
+
+  let(:entity) { described_class.new(export, request: double) }
+
+  subject { entity.as_json }
+
+  it 'has the correct attributes' do
+    expect(subject).to eq({
+      relation: export.relation,
+      status: export.status,
+      error: export.error,
+      updated_at: export.updated_at
+    })
+  end
+end

spec/lib/gitlab/import_export/command_line_util_spec.rb

@@ -35,4 +35,19 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
   it 'has the right mask for uploads' do
     expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555
   end
+
+  describe '#gzip' do
+    it 'compresses specified file' do
+      tempfile = Tempfile.new('test', path)
+      filename = File.basename(tempfile.path)
+
+      subject.gzip(dir: path, filename: filename)
+    end
+
+    context 'when exception occurs' do
+      it 'raises an exception' do
+        expect { subject.gzip(dir: path, filename: 'test') }.to raise_error(Gitlab::ImportExport::Error)
+      end
+    end
+  end
 end

spec/models/bulk_imports/exports/group_config_spec.rb

@@ -30,24 +30,6 @@ RSpec.describe BulkImports::Exports::GroupConfig do
     end
   end
 
-  describe '#validate_user_permissions' do
-    let_it_be(:user) { create(:user) }
-
-    context 'when user cannot admin project' do
-      it 'returns false' do
-        expect { subject.validate_user_permissions!(user) }.to raise_error(Gitlab::ImportExport::Error)
-      end
-    end
-
-    context 'when user can admin project' do
-      it 'returns true' do
-        exportable.add_owner(user)
-
-        expect(subject.validate_user_permissions!(user)).to eq(true)
-      end
-    end
-  end
-
   describe '#exportable_relations' do
     it 'returns a list of top level exportable relations' do
       expect(subject.exportable_relations).to include('milestones', 'badges', 'boards', 'labels')

spec/models/bulk_imports/exports/project_config_spec.rb

@@ -30,24 +30,6 @@ RSpec.describe BulkImports::Exports::ProjectConfig do
     end
   end
 
-  describe '#validate_user_permissions' do
-    let_it_be(:user) { create(:user) }
-
-    context 'when user cannot admin project' do
-      it 'returns false' do
-        expect { subject.validate_user_permissions!(user) }.to raise_error(Gitlab::ImportExport::Error)
-      end
-    end
-
-    context 'when user can admin project' do
-      it 'returns true' do
-        exportable.add_maintainer(user)
-
-        expect(subject.validate_user_permissions!(user)).to eq(true)
-      end
-    end
-  end
-
   describe '#exportable_relations' do
     it 'returns a list of top level exportable relations' do
       expect(subject.exportable_relations).to include('issues', 'labels', 'milestones', 'merge_requests')

spec/models/group_spec.rb

@@ -2390,4 +2390,12 @@ RSpec.describe Group do
 
     it { is_expected.to eq(Set.new([child_1.id])) }
   end
+
+  describe '#to_ability_name' do
+    it 'returns group' do
+      group = build(:group)
+
+      expect(group.to_ability_name).to eq('group')
+    end
+  end
 end

spec/requests/api/group_export_spec.rb

@@ -178,4 +178,74 @@ RSpec.describe API::GroupExport do
       end
     end
   end
+
+  describe 'relations export' do
+    let(:path) { "/groups/#{group.id}/export_relations" }
+    let(:download_path) { "/groups/#{group.id}/export_relations/download?relation=labels" }
+    let(:status_path) { "/groups/#{group.id}/export_relations/status" }
+
+    before do
+      stub_feature_flags(group_import_export: true)
+      group.add_owner(user)
+    end
+
+    describe 'POST /groups/:id/export_relations' do
+      it 'accepts the request' do
+        post api(path, user)
+
+        expect(response).to have_gitlab_http_status(:accepted)
+      end
+
+      context 'when response is not success' do
+        it 'returns api error' do
+          allow_next_instance_of(BulkImports::ExportService) do |service|
+            allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'error', http_status: :error))
+          end
+
+          post api(path, user)
+
+          expect(response).to have_gitlab_http_status(:error)
+        end
+      end
+    end
+
+    describe 'GET /groups/:id/export_relations/download' do
+      let(:export) { create(:bulk_import_export, group: group, relation: 'labels') }
+      let(:upload) { create(:bulk_import_export_upload, export: export) }
+
+      context 'when export file exists' do
+        it 'downloads exported group archive' do
+          upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz'))
+
+          get api(download_path, user)
+
+          expect(response).to have_gitlab_http_status(:ok)
+        end
+      end
+
+      context 'when export_file.file does not exist' do
+        it 'returns 404' do
+          allow(upload).to receive(:export_file).and_return(nil)
+
+          get api(download_path, user)
+
+          expect(response).to have_gitlab_http_status(:not_found)
+        end
+      end
+    end
+
+    describe 'GET /groups/:id/export_relations/status' do
+      it 'returns a list of relation export statuses' do
+        create(:bulk_import_export, :started, group: group, relation: 'labels')
+        create(:bulk_import_export, :finished, group: group, relation: 'milestones')
+        create(:bulk_import_export, :failed, group: group, relation: 'badges')
+
+        get api(status_path, user)
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'badges')
+        expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1)
+      end
+    end
+  end
 end

spec/services/bulk_imports/export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::ExportService do
+  let_it_be(:group) { create(:group) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    group.add_owner(user)
+  end
+
+  subject { described_class.new(exportable: group, user: user) }
+
+  describe '#execute' do
+    it 'schedules RelationExportWorker for each top level relation' do
+      expect(subject).to receive(:execute).and_return(ServiceResponse.success).and_call_original
+      top_level_relations = BulkImports::Export.config(group).exportable_relations
+
+      top_level_relations.each do |relation|
+        expect(BulkImports::RelationExportWorker)
+          .to receive(:perform_async)
+          .with(user.id, group.id, group.class.name, relation)
+      end
+
+      subject.execute
+    end
+
+    context 'when exception occurs' do
+      it 'does not schedule RelationExportWorker' do
+        service = described_class.new(exportable: nil, user: user)
+
+        expect(service)
+          .to receive(:execute)
+          .and_return(ServiceResponse.error(message: 'Gitlab::ImportExport::Error', http_status: :unprocessible_entity))
+          .and_call_original
+        expect(BulkImports::RelationExportWorker).not_to receive(:perform_async)
+
+        service.execute
+      end
+    end
+  end
+end

spec/services/bulk_imports/relation_export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::RelationExportService do
+  let_it_be(:jid) { 'jid' }
+  let_it_be(:relation) { 'labels' }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:label) { create(:group_label, group: group) }
+  let_it_be(:export_path) { "#{Dir.tmpdir}/relation_export_service_spec/tree" }
+  let_it_be_with_reload(:export) { create(:bulk_import_export, group: group, relation: relation) }
+
+  before do
+    group.add_owner(user)
+
+    allow(export).to receive(:export_path).and_return(export_path)
+  end
+
+  after :all do
+    FileUtils.rm_rf(export_path)
+  end
+
+  subject { described_class.new(user, group, relation, jid) }
+
+  describe '#execute' do
+    it 'exports specified relation and marks export as finished' do
+      subject.execute
+
+      expect(export.reload.upload.export_file).to be_present
+      expect(export.finished?).to eq(true)
+    end
+
+    it 'removes temp export files' do
+      subject.execute
+
+      expect(Dir.exist?(export_path)).to eq(false)
+    end
+
+    it 'exports specified relation and marks export as finished' do
+      subject.execute
+
+      expect(export.upload.export_file).to be_present
+    end
+
+    context 'when export record does not exist' do
+      let(:another_group) { create(:group) }
+
+      subject { described_class.new(user, another_group, relation, jid) }
+
+      it 'creates export record' do
+        another_group.add_owner(user)
+
+        expect { subject.execute }
+          .to change { another_group.bulk_import_exports.count }
+          .from(0)
+          .to(1)
+      end
+    end
+
+    context 'when there is existing export present' do
+      let(:upload) { create(:bulk_import_export_upload, export: export) }
+
+      it 'removes existing export before exporting' do
+        upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz'))
+
+        expect_any_instance_of(BulkImports::ExportUpload) do |upload|
+          expect(upload).to receive(:remove_export_file!)
+        end
+
+        subject.execute
+      end
+    end
+
+    context 'when exception occurs during export' do
+      shared_examples 'tracks exception' do |exception_class|
+        it 'tracks exception' do
+          expect(Gitlab::ErrorTracking)
+            .to receive(:track_exception)
+            .with(exception_class, exportable_id: group.id, exportable_type: group.class.name)
+            .and_call_original
+
+          subject.execute
+        end
+      end
+
+      before do
+        allow_next_instance_of(BulkImports::ExportUpload) do |upload|
+          allow(upload).to receive(:save!).and_raise(StandardError)
+        end
+      end
+
+      it 'marks export as failed' do
+        subject.execute
+
+        expect(export.reload.failed?).to eq(true)
+      end
+
+      include_examples 'tracks exception', StandardError
+
+      context 'when passed relation is not supported' do
+        let(:relation) { 'unsupported' }
+
+        include_examples 'tracks exception', ActiveRecord::RecordInvalid
+      end
+
+      context 'when user is not allowed to perform export' do
+        let(:another_user) { create(:user) }
+
+        subject { described_class.new(another_user, group, relation, jid) }
+
+        include_examples 'tracks exception', Gitlab::ImportExport::Error
+      end
+    end
+  end
+end

spec/workers/bulk_imports/relation_export_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::RelationExportWorker do
+  let_it_be(:jid) { 'jid' }
+  let_it_be(:relation) { 'labels' }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+
+  let(:job_args) { [user.id, group.id, group.class.name, relation] }
+
+  describe '#perform' do
+    include_examples 'an idempotent worker' do
+      context 'when export record does not exist' do
+        let(:another_group) { create(:group) }
+        let(:job_args) { [user.id, another_group.id, another_group.class.name, relation] }
+
+        it 'creates export record' do
+          another_group.add_owner(user)
+
+          expect { perform_multiple(job_args) }
+            .to change { another_group.bulk_import_exports.count }
+            .from(0)
+            .to(1)
+        end
+      end
+
+      it 'executes RelationExportService' do
+        group.add_owner(user)
+
+        service = instance_double(BulkImports::RelationExportService)
+
+        expect(BulkImports::RelationExportService)
+          .to receive(:new)
+          .with(user, group, relation, anything)
+          .twice
+          .and_return(service)
+        expect(service)
+          .to receive(:execute)
+          .twice
+
+        perform_multiple(job_args)
+      end
+    end
+  end
+end