Commit 9b0488ec authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch 'group-export-with-ndjson' into 'master'

Introduces Group export feature with ndjson

See merge request gitlab-org/gitlab!29590
parents d3c9c616 e04e8c14
......@@ -56,7 +56,7 @@ module Groups
end
def tree_exporter
Gitlab::ImportExport::Group::LegacyTreeSaver.new(
tree_exporter_class.new(
group: @group,
current_user: @current_user,
shared: @shared,
......@@ -64,6 +64,14 @@ module Groups
)
end
def tree_exporter_class
if ::Feature.enabled?(:group_import_export_ndjson, @group&.parent)
Gitlab::ImportExport::Group::TreeSaver
else
Gitlab::ImportExport::Group::LegacyTreeSaver
end
end
def file_saver
Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared)
end
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::Group::TreeSaver do
describe 'saves the group tree into a json object' do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:label) { create(:group_label) }
let_it_be(:parent_epic) { create(:epic, group: group) }
let_it_be(:epic) { create(:epic, group: group, parent: parent_epic) }
let_it_be(:epic_event) { create(:event, :created, target: epic, group: group, author: user) }
let_it_be(:epic_push_event) { create(:event, :pushed, target: epic, group: group, author: user) }
let_it_be(:milestone) { create(:milestone, group: group) }
let_it_be(:board) { create(:board, group: group, assignee: user, labels: [label]) }
let_it_be(:note) { create(:note, noteable: epic) }
let_it_be(:note_event) { create(:event, :created, target: note, author: user) }
let_it_be(:epic_emoji) { create(:award_emoji, awardable: epic) }
let_it_be(:epic_note_emoji) { create(:award_emoji, awardable: note) }
let(:shared) { Gitlab::ImportExport::Shared.new(group) }
let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec_ee" }
subject(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
before_all do
group.add_maintainer(user)
end
after do
FileUtils.rm_rf(export_path)
end
it 'saves successfully' do
expect_successful_save(group_tree_saver)
end
context 'epics relation' do
let(:epic_json) do
read_association(group, 'epics').find do |attrs|
attrs['id'] == epic.id
end
end
it 'saves top level epics' do
expect_successful_save(group_tree_saver)
expect(read_association(group, "epics").size).to eq(2)
end
it 'saves parent of epic' do
expect_successful_save(group_tree_saver)
parent = epic_json['parent']
expect(parent).not_to be_empty
expect(parent['id']).to eq(parent_epic.id)
end
it 'saves epic notes' do
expect_successful_save(group_tree_saver)
notes = epic_json['notes']
expect(notes).not_to be_empty
expect(notes.first['note']).to eq(note.note)
expect(notes.first['noteable_id']).to eq(epic.id)
end
it 'saves epic events' do
expect_successful_save(group_tree_saver)
events = epic_json['events']
expect(events).not_to be_empty
event_actions = events.map { |event| event['action'] }
expect(event_actions).to contain_exactly(epic_event.action, epic_push_event.action)
end
it "saves epic's note events" do
expect_successful_save(group_tree_saver)
notes = epic_json['notes']
expect(notes.first['events'].first['action']).to eq(note_event.action)
end
it "saves epic's award emojis" do
expect_successful_save(group_tree_saver)
award_emoji = epic_json['award_emoji'].first
expect(award_emoji['name']).to eq(epic_emoji.name)
end
it "saves epic's note award emojis" do
expect_successful_save(group_tree_saver)
award_emoji = epic_json['notes'].first['award_emoji'].first
expect(award_emoji['name']).to eq(epic_note_emoji.name)
end
end
context 'boards relation' do
before do
stub_licensed_features(board_assignee_lists: true, board_milestone_lists: true)
create(:list, board: board, user: user, list_type: List.list_types[:assignee], position: 0)
create(:list, board: board, milestone: milestone, list_type: List.list_types[:milestone], position: 1)
expect_successful_save(group_tree_saver)
end
it 'saves top level boards' do
expect(read_association(group, 'boards').size).to eq(1)
end
it 'saves board assignee' do
expect(read_association(group, 'boards').first['board_assignee']['assignee_id']).to eq(user.id)
end
it 'saves board labels' do
labels = read_association(group, 'boards').first['labels']
expect(labels).not_to be_empty
expect(labels.first['title']).to eq(label.title)
end
it 'saves board lists' do
lists = read_association(group, 'boards').first['lists']
expect(lists).not_to be_empty
milestone_list = lists.find { |list| list['list_type'] == 'milestone' }
assignee_list = lists.find { |list| list['list_type'] == 'assignee' }
expect(milestone_list['milestone_id']).to eq(milestone.id)
expect(assignee_list['user_id']).to eq(user.id)
end
end
it 'saves the milestone data when there are boards with predefined milestones' do
milestone = Milestone::Upcoming
board_with_milestone = create(:board, group: group, milestone_id: milestone.id)
expect_successful_save(group_tree_saver)
board_data = read_association(group, 'boards').find { |board| board['id'] == board_with_milestone.id }
expect(board_data).to include(
'milestone_id' => milestone.id,
'milestone' => {
'id' => milestone.id,
'name' => milestone.name,
'title' => milestone.title
}
)
end
it 'saves the milestone data when there are boards with persisted milestones' do
milestone = create(:milestone)
board_with_milestone = create(:board, group: group, milestone_id: milestone.id)
expect_successful_save(group_tree_saver)
board_data = read_association(group, 'boards').find { |board| board['id'] == board_with_milestone.id }
expect(board_data).to include(
'milestone_id' => milestone.id,
'milestone' => a_hash_including(
'id' => milestone.id,
'title' => milestone.title
)
)
end
end
def exported_path_for(file)
File.join(group_tree_saver.full_path, 'groups', file)
end
def read_association(group, association)
path = exported_path_for(File.join("#{group.id}", "#{association}.ndjson"))
File.foreach(path).map {|line| JSON.parse(line) }
end
def expect_successful_save(group_tree_saver)
expect(group_tree_saver.save).to be true
expect(group_tree_saver.shared.errors).to be_empty
end
end
......@@ -91,6 +91,10 @@ module Gitlab
def legacy_group_config_file
Rails.root.join('lib/gitlab/import_export/group/legacy_import_export.yml')
end
def group_config_file
Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
end
end
end
......
# Model relationships to be included in the group import/export
#
# This list _must_ only contain relationships that are available to both FOSS and
# Enterprise editions. EE specific relationships must be defined in the `ee` section further
# down below.
tree:
group:
- :milestones
- :badges
- labels:
- :priorities
- boards:
- lists:
- label:
- :priorities
- :board
- members:
- :user
included_attributes:
user:
- :id
- :email
- :username
author:
- :name
excluded_attributes:
group:
- :owner_id
- :created_at
- :updated_at
- :runners_token
- :runners_token_encrypted
- :saml_discovery_token
- :visibility_level
- :trial_ends_on
- :shared_runners_minute_limit
- :extra_shared_runners_minutes_limit
epics:
- :state_id
methods:
labels:
- :type
label:
- :type
badges:
- :type
notes:
- :type
events:
- :action
lists:
- :list_type
epics:
- :state
preloads:
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee:
tree:
group:
- epics:
- :parent
- :award_emoji
- events:
- :push_event_payload
- notes:
- :author
- :award_emoji
- events:
- :push_event_payload
- boards:
- :board_assignee
- :milestone
- labels:
- :priorities
- lists:
- milestone:
- events:
- :push_event_payload
# frozen_string_literal: true
module Gitlab
module ImportExport
module Group
class TreeSaver
attr_reader :full_path, :shared
def initialize(group:, current_user:, shared:, params: {})
@params = params
@current_user = current_user
@shared = shared
@group = group
@full_path = File.join(@shared.export_path, 'tree')
end
def save
all_groups = Enumerator.new do |group_ids|
groups.each do |group|
serialize(group)
group_ids << group.id
end
end
json_writer.write_relation_array('groups', '_all', all_groups)
true
rescue => e
@shared.error(e)
false
ensure
json_writer&.close
end
private
def groups
@groups ||= Gitlab::ObjectHierarchy
.new(::Group.where(id: @group.id))
.base_and_descendants(with_depth: true)
.order_by(:depth)
end
def serialize(group)
ImportExport::JSON::StreamingSerializer.new(
group,
group_tree,
json_writer,
exportable_path: "groups/#{group.id}"
).execute
end
def group_tree
@group_tree ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: group_config
).group_tree
end
def group_config
Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
end
def json_writer
@json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::Group::TreeSaver do
describe 'saves the group tree into a json object' do
let_it_be(:user) { create(:user) }
let_it_be(:group) { setup_groups }
let(:shared) { Gitlab::ImportExport::Shared.new(group) }
let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" }
subject(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
before_all do
group.add_maintainer(user)
end
before do
allow_next_instance_of(Gitlab::ImportExport) do |import_export|
allow(import_export).to receive(:storage_path).and_return(export_path)
end
end
after do
FileUtils.rm_rf(export_path)
end
it 'saves the group successfully' do
expect(group_tree_saver.save).to be true
end
it 'fails to export a group' do
allow_next_instance_of(Gitlab::ImportExport::JSON::NdjsonWriter) do |ndjson_writer|
allow(ndjson_writer).to receive(:write_relation_array).and_raise(RuntimeError, 'exception')
end
expect(shared).to receive(:error).with(RuntimeError).and_call_original
expect(group_tree_saver.save).to be false
end
context 'exported files' do
before do
group_tree_saver.save
end
it 'has one group per line' do
groups_catalog =
File.readlines(exported_path_for('_all.ndjson'))
.map { |line| Integer(line) }
expect(groups_catalog.size).to eq(3)
expect(groups_catalog).to eq([
group.id,
group.descendants.first.id,
group.descendants.first.descendants.first.id
])
end
it 'has a file per group' do
group.self_and_descendants.pluck(:id).each do |id|
group_attributes_file = exported_path_for("#{id}.json")
expect(File.exist?(group_attributes_file)).to be(true)
end
end
context 'group attributes file' do
let(:group_attributes_file) { exported_path_for("#{group.id}.json") }
let(:group_attributes) { ::JSON.parse(File.read(group_attributes_file)) }
it 'has a file for each group with its attributes' do
expect(group_attributes['description']).to eq(group.description)
expect(group_attributes['parent_id']).to eq(group.parent_id)
end
shared_examples 'excluded attributes' do
excluded_attributes = %w[
owner_id
created_at
updated_at
runners_token
runners_token_encrypted
saml_discovery_token
]
excluded_attributes.each do |excluded_attribute|
it 'does not contain excluded attribute' do
expect(group_attributes).not_to include(excluded_attribute => group.public_send(excluded_attribute))
end
end
end
include_examples 'excluded attributes'
end
it 'has a file for each group association' do
group.self_and_descendants do |g|
%w[
badges
boards
epics
labels
members
milestones
].each do |association|
path = exported_path_for("#{g.id}", "#{association}.ndjson")
expect(File.exist?(path)).to eq(true), "#{path} does not exist"
end
end
end
end
end
def exported_path_for(*file)
File.join(group_tree_saver.full_path, 'groups', *file)
end
def setup_groups
root = setup_group
subgroup = setup_group(parent: root)
setup_group(parent: subgroup)
root
end
def setup_group(parent: nil)
group = create(:group, description: 'description', parent: parent)
create(:milestone, group: group)
create(:group_badge, group: group)
group_label = create(:group_label, group: group)
board = create(:board, group: group, milestone_id: Milestone::Upcoming.id)
create(:list, board: board, label: group_label)
create(:group_badge, group: group)
create(:label_priority, label: group_label, priority: 1)
group
end
end
......@@ -11,7 +11,7 @@ describe Groups::ImportExport::ExportService do
let(:export_service) { described_class.new(group: group, user: user) }
it 'enqueues an export job' do
expect(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {})
allow(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {})
export_service.async_execute
end
......@@ -49,7 +49,17 @@ describe Groups::ImportExport::ExportService do
FileUtils.rm_rf(archive_path)
end
it 'saves the models' do
it 'saves the models using ndjson tree saver' do
stub_feature_flags(group_import_export_ndjson: true)
expect(Gitlab::ImportExport::Group::TreeSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the models using legacy tree saver' do
stub_feature_flags(group_import_export_ndjson: false)
expect(Gitlab::ImportExport::Group::LegacyTreeSaver).to receive(:new).and_call_original
service.execute
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment