Merge branch 'georgekoltsov/add-group-relation-factory-and-object-builder' into 'master'

Add Group Import functionality

See merge request gitlab-org/gitlab!22384

app/models/ci/pipeline.rb

@@ -578,7 +578,7 @@ module Ci
     # Manually set the notes for a Ci::Pipeline
     # There is no ActiveRecord relation between Ci::Pipeline and notes
     # as they are related to a commit sha. This method helps importing
-    # them using the +Gitlab::ImportExport::RelationFactory+ class.
+    # them using the +Gitlab::ImportExport::ProjectRelationFactory+ class.
     def notes=(notes)
       notes.each do |note|
         note[:id] = nil

app/services/groups/import_export/import_service.rb

+# frozen_string_literal: true
+
+module Groups
+  module ImportExport
+    class ImportService
+      attr_reader :current_user, :group, :params
+
+      def initialize(group:, user:)
+        @group = group
+        @current_user = user
+        @shared = Gitlab::ImportExport::Shared.new(@group)
+      end
+
+      def execute
+        validate_user_permissions
+
+        if import_file && restorer.restore
+          @group
+        else
+          raise StandardError.new(@shared.errors.to_sentence)
+        end
+      rescue => e
+        raise StandardError.new(e.message)
+      ensure
+        remove_import_file
+      end
+
+      private
+
+      def import_file
+        @import_file ||= Gitlab::ImportExport::FileImporter.import(importable: @group,
+                                                                   archive_file: nil,
+                                                                   shared: @shared)
+      end
+
+      def restorer
+        @restorer ||= Gitlab::ImportExport::GroupTreeRestorer.new(user: @current_user,
+                                                                  shared: @shared,
+                                                                  group: @group,
+                                                                  group_hash: nil)
+      end
+
+      def remove_import_file
+        upload = @group.import_export_upload
+
+        return unless upload&.import_file&.file
+
+        upload.remove_import_file!
+        upload.save!
+      end
+
+      def validate_user_permissions
+        unless current_user.can?(:admin_group, group)
+          raise ::Gitlab::ImportExport::Error.new(
+            "User with ID: %s does not have permission to Group %s with ID: %s." %
+              [current_user.id, group.name, group.id])
+        end
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -137,6 +137,7 @@
 - gitlab_shell
 - group_destroy
 - group_export
+- group_import
 - import_issues_csv
 - invalid_gpg_signature_update
 - irker

app/workers/group_export_worker.rb

@@ -4,11 +4,11 @@ class GroupExportWorker
   include ApplicationWorker
   include ExceptionBacktrace
 
-  feature_category :source_code_management
+  feature_category :importers
 
   def perform(current_user_id, group_id, params = {})
     current_user = User.find(current_user_id)
-    group        = Group.find(group_id)
+    group = Group.find(group_id)
 
     ::Groups::ImportExport::ExportService.new(group: group, user: current_user, params: params).execute
   end

app/workers/group_import_worker.rb

+# frozen_string_literal: true
+
+class GroupImportWorker
+  include ApplicationWorker
+  include ExceptionBacktrace
+
+  feature_category :importers
+
+  def perform(user_id, group_id)
+    current_user = User.find(user_id)
+    group = Group.find(group_id)
+
+    ::Groups::ImportExport::ImportService.new(group: group, user: current_user).execute
+  end
+end

config/sidekiq_queues.yml

@@ -110,6 +110,8 @@
   - 1
 - - group_export
   - 1
+- - group_import
+  - 1
 - - hashed_storage
   - 1
 - - import_issues_csv

ee/spec/lib/gitlab/import_export/group_relation_factory_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::GroupRelationFactory do
+  let(:group) { create(:group) }
+  let(:members_mapper) { double('members_mapper').as_null_object }
+  let(:user) { create(:admin) }
+  let(:excluded_keys) { [] }
+  let(:created_object) do
+    described_class.create(relation_sym:   relation_sym,
+                           relation_hash:  relation_hash,
+                           members_mapper: members_mapper,
+                           object_builder: Gitlab::ImportExport::GroupObjectBuilder,
+                           user:           user,
+                           importable:     group,
+                           excluded_keys:  excluded_keys)
+  end
+
+  context 'epic object' do
+    let(:relation_sym) { :epics }
+    let(:id) { random_id }
+    let(:original_group_id) { random_id }
+
+    let(:relation_hash) do
+      {
+        'id'           => id,
+        'milestone_id' => nil,
+        'group_id'     => original_group_id,
+        'assignee_id'  => nil,
+        'created_at'   => 'Wed, 20 Nov 2019 11:02:24 UTC +00:00',
+        'updated_at'   => 'Wed, 20 Nov 2019 11:02:24 UTC +00:00',
+        'title'        => 'Title',
+        'description'  => 'Description',
+        'state_id'     => 'opened'
+      }
+    end
+
+    it 'does not have the original ID' do
+      expect(created_object.id).not_to eq(id)
+    end
+
+    it 'does not have the original group_id' do
+      expect(created_object.group_id).not_to eq(original_group_id)
+    end
+
+    it 'has the new group_id' do
+      expect(created_object.group_id).to eq(group.id)
+    end
+
+    context 'excluded attributes' do
+      let(:excluded_keys) { %w[description] }
+
+      it 'are removed from the imported object' do
+        expect(created_object.description).to be_nil
+      end
+    end
+
+    context 'overridden model' do
+      let(:relation_sym) { :parent }
+
+      it 'does not raise errors' do
+        expect { created_object }.not_to raise_error
+      end
+    end
+  end
+
+  context 'Notes user references' do
+    let(:relation_sym) { :notes }
+    let(:new_user) { create(:user) }
+    let(:exported_member) do
+      {
+        'id'                 => 111,
+        'access_level'       => 30,
+        'source_id'          => 1,
+        'source_type'        => 'Namespace',
+        'user_id'            => 3,
+        'notification_level' => 3,
+        'created_at'         => '2016-11-18T09:29:42.634Z',
+        'updated_at'         => '2016-11-18T09:29:42.634Z',
+        'user'               => {
+          'id'       => 999,
+          'email'    => new_user.email,
+          'username' => new_user.username
+        }
+      }
+    end
+
+    let(:relation_hash) do
+      {
+        'id'            => 4947,
+        'note'          => 'note',
+        'noteable_type' => 'Epic',
+        'author_id'     => 999,
+        'created_at'    => '2016-11-18T09:29:42.634Z',
+        'updated_at'    => '2016-11-18T09:29:42.634Z',
+        'project_id'    => 1,
+        'attachment'    => {
+          'url' => nil
+        },
+        'noteable_id'   => 377,
+        'system'        => true,
+        'author'        => {
+          'name' => 'Administrator'
+        },
+        'events' => []
+      }
+    end
+
+    let(:members_mapper) do
+      Gitlab::ImportExport::MembersMapper.new(
+        exported_members: [exported_member],
+        user:             user,
+        importable:       group)
+    end
+
+    it 'maps the right author to the imported note' do
+      expect(created_object.author).to eq(new_user)
+    end
+  end
+
+  def random_id
+    rand(1000..10000)
+  end
+end

lib/gitlab/import_export/base_relation_factory.rb

@@ -24,7 +24,8 @@ module Gitlab
          last_edited_by_id
          merge_user_id
          resolved_by_id
-         closed_by_id owner_id
+         closed_by_id
+         owner_id
        ].freeze
 
       TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze

lib/gitlab/import_export/group_object_builder.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    # Given a class, it finds or creates a new object at group level.
+    #
+    # Example:
+    #   `GroupObjectBuilder.build(Label, label_attributes)`
+    #    finds or initializes a label with the given attributes.
+    class GroupObjectBuilder < BaseObjectBuilder
+      def self.build(*args)
+        Group.transaction do
+          super
+        end
+      end
+
+      def initialize(klass, attributes)
+        super
+
+        @group = @attributes['group']
+      end
+
+      private
+
+      attr_reader :group
+
+      def where_clauses
+        [
+          where_clause_base,
+          where_clause_for_title,
+          where_clause_for_description,
+          where_clause_for_created_at
+        ].compact
+      end
+
+      # Returns Arel clause `"{table_name}"."group_id" = {group.id}`
+      def where_clause_base
+        table[:group_id].in(group_and_ancestor_ids)
+      end
+
+      def group_and_ancestor_ids
+        group.ancestors.map(&:id) << group.id
+      end
+    end
+  end
+end

lib/gitlab/import_export/group_relation_factory.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class GroupRelationFactory < BaseRelationFactory
+      OVERRIDES = {
+        labels:     :group_labels,
+        priorities: :label_priorities,
+        label:      :group_label,
+        parent:     :epic
+      }.freeze
+
+      EXISTING_OBJECT_RELATIONS = %i[
+        epic
+        epics
+        milestone
+        milestones
+        label
+        labels
+        group_label
+        group_labels
+      ].freeze
+
+      private
+
+      def setup_models
+        setup_note if @relation_name == :notes
+
+        update_group_references
+      end
+
+      def update_group_references
+        return unless self.class.existing_object_relations.include?(@relation_name)
+        return unless @relation_hash['group_id']
+
+        @relation_hash['group_id'] = @importable.id
+      end
+    end
+  end
+end

lib/gitlab/import_export/group_tree_restorer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class GroupTreeRestorer
+      attr_reader :user
+      attr_reader :shared
+      attr_reader :group
+
+      def initialize(user:, shared:, group:, group_hash:)
+        @path = File.join(shared.export_path, 'group.json')
+        @user = user
+        @shared = shared
+        @group = group
+        @group_hash = group_hash
+      end
+
+      def restore
+        @tree_hash = @group_hash || read_tree_hash
+        @group_members = @tree_hash.delete('members')
+        @children = @tree_hash.delete('children')
+
+        if members_mapper.map && restorer.restore
+          @children&.each do |group_hash|
+            group = create_group(group_hash: group_hash, parent_group: @group)
+            shared = Gitlab::ImportExport::Shared.new(group)
+
+            self.class.new(
+              user: @user,
+              shared: shared,
+              group: group,
+              group_hash: group_hash
+            ).restore
+          end
+        end
+
+        return false if @shared.errors.any?
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def read_tree_hash
+        json = IO.read(@path)
+        ActiveSupport::JSON.decode(json)
+      rescue => e
+        @shared.logger.error(
+          group_id:   @group.id,
+          group_name: @group.name,
+          message:    "Import/Export error: #{e.message}"
+        )
+
+        raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+      end
+
+      def restorer
+        @relation_tree_restorer ||= RelationTreeRestorer.new(
+          user:             @user,
+          shared:           @shared,
+          importable:       @group,
+          tree_hash:        @tree_hash.except('name', 'path'),
+          members_mapper:   members_mapper,
+          object_builder:   object_builder,
+          relation_factory: relation_factory,
+          reader:           reader
+        )
+      end
+
+      def create_group(group_hash:, parent_group:)
+        group_params = {
+          name:      group_hash['name'],
+          path:      group_hash['path'],
+          parent_id: parent_group&.id
+        }
+
+        ::Groups::CreateService.new(@user, group_params).execute
+      end
+
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
+      end
+
+      def relation_factory
+        Gitlab::ImportExport::GroupRelationFactory
+      end
+
+      def object_builder
+        Gitlab::ImportExport::GroupObjectBuilder
+      end
+
+      def reader
+        @reader ||= Gitlab::ImportExport::Reader.new(
+          shared: @shared,
+          config: Gitlab::ImportExport::Config.new(
+            config: Gitlab::ImportExport.group_config_file
+          ).to_h
+        )
+      end
+    end
+  end
+end

lib/gitlab/import_export/members_mapper.rb

@@ -9,7 +9,7 @@ module Gitlab
         @importable = importable
 
         # This needs to run first, as second call would be from #map
-        # which means project members already exist.
+        # which means Project/Group members already exist.
         ensure_default_member!
       end
 
@@ -47,6 +47,8 @@ module Gitlab
       end
 
       def ensure_default_member!
+        return if user_already_member?
+
         @importable.members.destroy_all # rubocop: disable DestroyAll
 
         relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true)
@@ -54,6 +56,12 @@ module Gitlab
         raise e, "Error adding importer user to #{@importable.class} members. #{e.message}"
       end
 
+      def user_already_member?
+        member = @importable.members&.first
+
+        member&.user == @user && member.access_level >= relation_class::MAINTAINER
+      end
+
       def add_team_member(member, existing_user = nil)
         member['user'] = existing_user
 

spec/lib/gitlab/import_export/group_object_builder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::GroupObjectBuilder do
+  let(:group) { create(:group) }
+  let(:base_attributes) do
+    {
+      'title'       => 'title',
+      'description' => 'description',
+      'group'       => group
+    }
+  end
+
+  context 'labels' do
+    let(:label_attributes) { base_attributes.merge('type' => 'GroupLabel') }
+
+    it 'finds the existing group label' do
+      group_label = create(:group_label, base_attributes)
+
+      expect(described_class.build(Label, label_attributes)).to eq(group_label)
+    end
+
+    it 'creates a new label' do
+      label = described_class.build(Label, label_attributes)
+
+      expect(label.persisted?).to be true
+    end
+  end
+
+  context 'milestones' do
+    it 'finds the existing group milestone' do
+      milestone = create(:milestone, base_attributes)
+
+      expect(described_class.build(Milestone, base_attributes)).to eq(milestone)
+    end
+
+    it 'creates a new milestone' do
+      milestone = described_class.build(Milestone, base_attributes)
+
+      expect(milestone.persisted?).to be true
+    end
+  end
+end

spec/lib/gitlab/import_export/group_relation_factory_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::GroupRelationFactory do
+  let(:group) { create(:group) }
+  let(:members_mapper) { double('members_mapper').as_null_object }
+  let(:user) { create(:admin) }
+  let(:excluded_keys) { [] }
+  let(:created_object) do
+    described_class.create(relation_sym:   relation_sym,
+                           relation_hash:  relation_hash,
+                           members_mapper: members_mapper,
+                           object_builder: Gitlab::ImportExport::GroupObjectBuilder,
+                           user:           user,
+                           importable:     group,
+                           excluded_keys:  excluded_keys)
+  end
+
+  context 'label object' do
+    let(:relation_sym) { :group_label }
+    let(:id) { random_id }
+    let(:original_group_id) { random_id }
+
+    let(:relation_hash) do
+      {
+        'id'          => 123456,
+        'title'       => 'Bruffefunc',
+        'color'       => '#1d2da4',
+        'project_id'  => nil,
+        'created_at'  => '2019-11-20T17:02:20.546Z',
+        'updated_at'  => '2019-11-20T17:02:20.546Z',
+        'template'    => false,
+        'description' => 'Description',
+        'group_id'    => original_group_id,
+        'type'        => 'GroupLabel',
+        'priorities'  => [],
+        'textColor'   => '#FFFFFF'
+      }
+    end
+
+    it 'does not have the original ID' do
+      expect(created_object.id).not_to eq(id)
+    end
+
+    it 'does not have the original group_id' do
+      expect(created_object.group_id).not_to eq(original_group_id)
+    end
+
+    it 'has the new group_id' do
+      expect(created_object.group_id).to eq(group.id)
+    end
+
+    context 'excluded attributes' do
+      let(:excluded_keys) { %w[description] }
+
+      it 'are removed from the imported object' do
+        expect(created_object.description).to be_nil
+      end
+    end
+  end
+
+  context 'Notes user references' do
+    let(:relation_sym) { :notes }
+    let(:new_user) { create(:user) }
+    let(:exported_member) do
+      {
+        'id'                 => 111,
+        'access_level'       => 30,
+        'source_id'          => 1,
+        'source_type'        => 'Namespace',
+        'user_id'            => 3,
+        'notification_level' => 3,
+        'created_at'         => '2016-11-18T09:29:42.634Z',
+        'updated_at'         => '2016-11-18T09:29:42.634Z',
+        'user'               => {
+          'id'       => 999,
+          'email'    => new_user.email,
+          'username' => new_user.username
+        }
+      }
+    end
+
+    let(:relation_hash) do
+      {
+        'id'            => 4947,
+        'note'          => 'note',
+        'noteable_type' => 'Epic',
+        'author_id'     => 999,
+        'created_at'    => '2016-11-18T09:29:42.634Z',
+        'updated_at'    => '2016-11-18T09:29:42.634Z',
+        'project_id'    => 1,
+        'attachment'    => {
+          'url' => nil
+        },
+        'noteable_id'   => 377,
+        'system'        => true,
+        'author'        => {
+          'name' => 'Administrator'
+        },
+        'events' => []
+      }
+    end
+
+    let(:members_mapper) do
+      Gitlab::ImportExport::MembersMapper.new(
+        exported_members: [exported_member],
+        user:             user,
+        importable:       group)
+    end
+
+    it 'maps the right author to the imported note' do
+      expect(created_object.author).to eq(new_user)
+    end
+  end
+
+  def random_id
+    rand(1000..10000)
+  end
+end

spec/lib/gitlab/import_export/group_tree_restorer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::GroupTreeRestorer do
+  include ImportExport::CommonUtil
+
+  let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+
+  describe 'restore group tree' do
+    before(:context) do
+      # Using an admin for import, so we can check assignment of existing members
+      user = create(:admin, username: 'root')
+      create(:user, username: 'adriene.mcclure')
+      create(:user, username: 'gwendolyn_robel')
+
+      RSpec::Mocks.with_temporary_scope do
+        @group = create(:group, name: 'group', path: 'group')
+        @shared = Gitlab::ImportExport::Shared.new(@group)
+
+        setup_import_export_config('group_exports/complex')
+
+        group_tree_restorer = described_class.new(user: user, shared: @shared, group: @group, group_hash: nil)
+
+        @restored_group_json = group_tree_restorer.restore
+      end
+    end
+
+    context 'JSON' do
+      it 'restores models based on JSON' do
+        expect(@restored_group_json).to be_truthy
+      end
+
+      it 'has the group description' do
+        expect(Group.find_by_path('group').description).to eq('Group Description')
+      end
+
+      it 'has group labels' do
+        expect(@group.labels.count).to eq(10)
+      end
+
+      it 'has issue boards' do
+        expect(@group.boards.count).to eq(2)
+      end
+
+      it 'has badges' do
+        expect(@group.badges.count).to eq(1)
+      end
+
+      it 'has milestones' do
+        expect(@group.milestones.count).to eq(5)
+      end
+
+      it 'has group children' do
+        expect(@group.children.count).to eq(2)
+      end
+
+      it 'has group members' do
+        expect(@group.members.map(&:user).map(&:username)).to contain_exactly('root', 'adriene.mcclure', 'gwendolyn_robel')
+      end
+    end
+  end
+
+  context 'group.json file access check' do
+    let(:user) { create(:user) }
+    let!(:group) { create(:group, name: 'group2', path: 'group2') }
+    let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group, group_hash: nil) }
+    let(:restored_group_json) { group_tree_restorer.restore }
+
+    it 'does not read a symlink' do
+      Dir.mktmpdir do |tmpdir|
+        setup_symlink(tmpdir, 'group.json')
+        allow(shared).to receive(:export_path).and_call_original
+
+        expect(group_tree_restorer.restore).to eq(false)
+        expect(shared.errors).to include('Incorrect JSON format')
+      end
+    end
+  end
+end

spec/services/groups/import_export/import_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Groups::ImportExport::ImportService do
+  describe '#execute' do
+    let(:user) { create(:admin) }
+    let(:group) { create(:group) }
+    let(:service) { described_class.new(group: group, user: user) }
+    let(:import_file) { fixture_file_upload('spec/fixtures/group_export.tar.gz') }
+
+    subject { service.execute }
+
+    before do
+      ImportExportUpload.create(group: group, import_file: import_file)
+    end
+
+    context 'when user has correct permissions' do
+      it 'imports group structure successfully' do
+        expect(subject).to be_truthy
+      end
+
+      it 'removes import file' do
+        subject
+
+        expect(group.import_export_upload.import_file.file).to be_nil
+      end
+    end
+
+    context 'when user does not have correct permissions' do
+      let(:user) { create(:user) }
+
+      it 'raises exception' do
+        expect { subject }.to raise_error(StandardError)
+      end
+    end
+  end
+end

spec/workers/group_import_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe GroupImportWorker do
+  let!(:user) { create(:user) }
+  let!(:group) { create(:group) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    context 'when it succeeds' do
+      it 'calls the ImportService' do
+        expect_any_instance_of(::Groups::ImportExport::ImportService).to receive(:execute)
+
+        subject.perform(user.id, group.id)
+      end
+    end
+
+    context 'when it fails' do
+      it 'raises an exception when params are invalid' do
+        expect_any_instance_of(::Groups::ImportExport::ImportService).not_to receive(:execute)
+
+        expect { subject.perform(1234, group.id) }.to raise_exception(ActiveRecord::RecordNotFound)
+        expect { subject.perform(user.id, 1234) }.to raise_exception(ActiveRecord::RecordNotFound)
+      end
+    end
+  end
+end