Merge branch 'jcunha-add-relations-export-api-for-projects' into 'master'

Add relations export API for Projects

See merge request gitlab-org/gitlab!70330

app/models/project.rb

@@ -233,6 +233,7 @@ class Project < ApplicationRecord
   has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
   has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :export_jobs, class_name: 'ProjectExportJob'
+  has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :project
   has_one :project_repository, inverse_of: :project
   has_one :tracing_setting, class_name: 'ProjectTracingSetting'
   has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting'

config/initializers/grape_validators.rb

@@ -10,3 +10,4 @@ Grape::Validations.register_validator(:check_assignees_count, ::API::Validations
 Grape::Validations.register_validator(:untrusted_regexp, ::API::Validations::Validators::UntrustedRegexp)
 Grape::Validations.register_validator(:email_or_email_list, ::API::Validations::Validators::EmailOrEmailList)
 Grape::Validations.register_validator(:iteration_id, ::API::Validations::Validators::IntegerOrCustomValue)
+Grape::Validations.register_validator(:project_portable, ::API::Validations::Validators::ProjectPortable)

doc/api/project_relations_export.md

+---
+stage: Manage
+group: Import
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+---
+
+# Project Relations Export API **(FREE)**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70330) in GitLab 14.4 behind the `bulk_import` [feature flag](../administration/feature_flags.md), disabled by default.
+
+FLAG:
+On GitLab.com, this feature is available.
+On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to
+[disable the `bulk_import` flag](../administration/feature_flags.md).
+The feature is not ready for production use. It is still in experimental stage and might change in the future.
+
+With the Project Relations Export API, you can partially export project structure. This API is
+similar to [project export](project_import_export.md),
+but it exports each top-level relation (for example, milestones/boards/labels) as a separate file
+instead of one archive. The project relations export API is primarily used in
+[group migration](../user/group/import/index.md#enable-or-disable-gitlab-group-migration)
+to support group project import.
+
+## Schedule new export
+
+Start a new project relations export:
+
+```plaintext
+POST /projects/:id/export_relations
+```
+
+| Attribute | Type           | Required | Description                              |
+| --------- | -------------- | -------- | ---------------------------------------- |
+| `id`      | integer/string | yes      | ID of the project owned by the authenticated user. |
+
+```shell
+curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/export_relations"
+```
+
+```json
+{
+  "message": "202 Accepted"
+}
+```
+
+## Export status
+
+View the status of the relations export:
+
+```plaintext
+GET /projects/:id/export_relations/status
+```
+
+| Attribute | Type           | Required | Description                              |
+| --------- | -------------- | -------- | ---------------------------------------- |
+| `id`      | integer/string | yes      | ID of the project owned by the authenticated user. |
+
+```shell
+curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" \
+     "https://gitlab.example.com/api/v4/projects/1/export_relations/status"
+```
+
+The status can be one of the following:
+
+- `0`: `started`
+- `1`: `finished`
+- `-1`: `failed`
+
+- `0` - `started`
+- `1` - `finished`
+- `-1` - `failed`
+
+```json
+[
+  {
+    "relation": "project_badges",
+    "status": 1,
+    "error": null,
+    "updated_at": "2021-05-04T11:25:20.423Z"
+  },
+  {
+    "relation": "boards",
+    "status": 1,
+    "error": null,
+    "updated_at": "2021-05-04T11:25:20.085Z"
+  }
+]
+```
+
+## Export download
+
+Download the finished relations export:
+
+```plaintext
+GET /projects/:id/export_relations/download
+```
+
+| Attribute       | Type           | Required | Description                              |
+| --------------- | -------------- | -------- | ---------------------------------------- |
+| `id`            | integer/string | yes      | ID of the project owned by the authenticated user. |
+| `relation`      | string         | yes      | Name of the project top-level relation to download. |
+
+```shell
+curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name \
+     --remote-name "https://gitlab.example.com/api/v4/projects/1/export_relations/download?relation=labels"
+```
+
+```shell
+ls labels.ndjson.gz
+labels.ndjson.gz
+```

lib/api/project_export.rb

@@ -74,6 +74,52 @@ module API
 
         accepted!
       end
+
+      resource do
+        before do
+          not_found! unless ::Feature.enabled?(:bulk_import, default_enabled: :yaml)
+        end
+
+        desc 'Start relations export' do
+          detail 'This feature was introduced in GitLab 14.4'
+        end
+        post ':id/export_relations' do
+          response = ::BulkImports::ExportService.new(portable: user_project, user: current_user).execute
+
+          if response.success?
+            accepted!
+          else
+            render_api_error!(message: 'Project relations export could not be started.')
+          end
+        end
+
+        desc 'Download relations export' do
+          detail 'This feature was introduced in GitLab 14.4'
+        end
+        params do
+          requires :relation,
+                   type: String,
+                   project_portable: true,
+                   desc: 'Project relation name'
+        end
+        get ':id/export_relations/download' do
+          export = user_project.bulk_import_exports.find_by_relation(params[:relation])
+          file = export&.upload&.export_file
+
+          if file
+            present_carrierwave_file!(file)
+          else
+            render_api_error!('404 Not found', 404)
+          end
+        end
+
+        desc 'Relations export status' do
+          detail 'This feature was introduced in GitLab 14.4'
+        end
+        get ':id/export_relations/status' do
+          present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus
+        end
+      end
     end
   end
 end

lib/api/validations/validators/project_portable.rb

+# frozen_string_literal: true
+
+module API
+  module Validations
+    module Validators
+      class ProjectPortable < Grape::Validations::Base
+        def validate_param!(attr_name, params)
+          portable = params[attr_name]
+
+          portable_relations = ::BulkImports::FileTransfer.config_for(::Project.new).portable_relations
+          return if portable_relations.include?(portable)
+
+          raise Grape::Exceptions::Validation.new(
+            params: [@scope.full_name(attr_name)],
+            message: "is not portable"
+          )
+        end
+      end
+    end
+  end
+end

spec/lib/api/validations/validators/project_portable_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Validations::Validators::ProjectPortable do
+  include ApiValidatorsHelpers
+
+  let(:portable) { 'labels' }
+  let(:not_portable) { 'project_members' }
+
+  subject do
+    described_class.new(['test'], {}, false, scope.new)
+  end
+
+  context 'valid portable' do
+    it 'does not raise a validation error' do
+      expect_no_validation_error('test' => portable)
+    end
+  end
+
+  context 'empty params' do
+    it 'raises a validation error' do
+      expect_validation_error('test' => nil)
+      expect_validation_error('test' => '')
+    end
+  end
+
+  context 'not portable' do
+    it 'raises a validation error' do
+      expect_validation_error('test' => not_portable) # Sha length > 40
+    end
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -595,6 +595,7 @@ project:
 - pending_builds
 - security_scans
 - ci_feature_usages
+- bulk_import_exports
 award_emoji:
 - awardable
 - user

spec/models/group_spec.rb

@@ -36,6 +36,7 @@ RSpec.describe Group do
     it { is_expected.to have_many(:debian_distributions).class_name('Packages::Debian::GroupDistribution').dependent(:destroy) }
     it { is_expected.to have_many(:daily_build_group_report_results).class_name('Ci::DailyBuildGroupReportResult') }
     it { is_expected.to have_many(:group_callouts).class_name('Users::GroupCallout').with_foreign_key(:group_id) }
+    it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') }
 
     describe '#members & #requesters' do
       let(:requester) { create(:user) }

spec/models/project_spec.rb

@@ -140,6 +140,7 @@ RSpec.describe Project, factory_default: :keep do
     it { is_expected.to have_many(:error_tracking_client_keys).class_name('ErrorTracking::ClientKey') }
     it { is_expected.to have_many(:pending_builds).class_name('Ci::PendingBuild') }
     it { is_expected.to have_many(:ci_feature_usages).class_name('Projects::CiFeatureUsage') }
+    it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') }
 
     # GitLab Pages
     it { is_expected.to have_many(:pages_domains) }

spec/requests/api/project_export_spec.rb

@@ -457,4 +457,143 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
       end
     end
   end
+
+  describe 'export relations' do
+    let(:relation) { 'labels' }
+    let(:download_path) { "/projects/#{project.id}/export_relations/download?relation=#{relation}" }
+    let(:path) { "/projects/#{project.id}/export_relations" }
+
+    let_it_be(:status_path) { "/projects/#{project.id}/export_relations/status" }
+
+    context 'when user is a maintainer' do
+      before do
+        project.add_maintainer(user)
+      end
+
+      describe 'POST /projects/:id/export_relations' do
+        it 'accepts the request' do
+          post api(path, user)
+
+          expect(response).to have_gitlab_http_status(:accepted)
+        end
+
+        context 'when response is not success' do
+          it 'returns api error' do
+            allow_next_instance_of(BulkImports::ExportService) do |service|
+              allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'error', http_status: :error))
+            end
+
+            post api(path, user)
+
+            expect(response).to have_gitlab_http_status(:error)
+          end
+        end
+      end
+
+      describe 'GET /projects/:id/export_relations/download' do
+        let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') }
+        let_it_be(:upload) { create(:bulk_import_export_upload, export: export) }
+
+        context 'when export file exists' do
+          it 'downloads exported project relation archive' do
+            upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz'))
+
+            get api(download_path, user)
+
+            expect(response).to have_gitlab_http_status(:ok)
+            expect(response.header['Content-Disposition']).to eq("attachment; filename=\"labels.ndjson.gz\"; filename*=UTF-8''labels.ndjson.gz")
+          end
+        end
+
+        context 'when relation is not portable' do
+          let(:relation) { ::BulkImports::FileTransfer::ProjectConfig.new(project).skipped_relations.first }
+
+          it_behaves_like '400 response' do
+            let(:request) { get api(download_path, user) }
+          end
+        end
+
+        context 'when export file does not exist' do
+          it 'returns 404' do
+            allow(upload).to receive(:export_file).and_return(nil)
+
+            get api(download_path, user)
+
+            expect(response).to have_gitlab_http_status(:not_found)
+          end
+        end
+      end
+
+      describe 'GET /projects/:id/export_relations/status' do
+        it 'returns a list of relation export statuses' do
+          create(:bulk_import_export, :started, project: project, relation: 'labels')
+          create(:bulk_import_export, :finished, project: project, relation: 'milestones')
+          create(:bulk_import_export, :failed, project: project, relation: 'project_badges')
+
+          get api(status_path, user)
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'project_badges')
+          expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1)
+        end
+      end
+
+      context 'with bulk_import FF disabled' do
+        before do
+          stub_feature_flags(bulk_import: false)
+        end
+
+        describe 'POST /projects/:id/export_relations' do
+          it_behaves_like '404 response' do
+            let(:request) { post api(path, user) }
+          end
+        end
+
+        describe 'GET /projects/:id/export_relations/download' do
+          let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') }
+          let_it_be(:upload) { create(:bulk_import_export_upload, export: export) }
+
+          before do
+            upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz'))
+          end
+
+          it_behaves_like '404 response' do
+            let(:request) { post api(path, user) }
+          end
+        end
+
+        describe 'GET /projects/:id/export_relations/status' do
+          it_behaves_like '404 response' do
+            let(:request) { get api(status_path, user) }
+          end
+        end
+      end
+    end
+
+    context 'when user is a developer' do
+      let_it_be(:developer) { create(:user) }
+
+      before do
+        project.add_developer(developer)
+      end
+
+      describe 'POST /projects/:id/export_relations' do
+        it_behaves_like '403 response' do
+          let(:request) { post api(path, developer) }
+        end
+      end
+
+      describe 'GET /projects/:id/export_relations/download' do
+        it_behaves_like '403 response' do
+          let(:request) { get api(download_path, developer) }
+        end
+      end
+
+      describe 'GET /projects/:id/export_relations/status' do
+        it_behaves_like '403 response' do
+          let(:request) { get api(status_path, developer) }
+        end
+      end
+    end
+  end
 end