Skip to content

G
gitlab-ce
Commit 2915c8be authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files
Options

Merge branch 'refactor-modules-for-import-export' into 'master' 

Refactor import_export

Closes #207846

See merge request gitlab-org/gitlab!25623
parents 0b29e5a9 06801192
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1331 additions and 1306 deletions
app/models/ci/pipeline.rb
View file @ 2915c8be
......@@ -599,7 +599,7 @@ module Ci
# Manually set the notes for a Ci::Pipeline
# There is no ActiveRecord relation between Ci::Pipeline and notes
# as they are related to a commit sha. This method helps importing
# them using the +Gitlab::ImportExport::ProjectRelationFactory+ class.
# them using the +Gitlab::ImportExport::Project::RelationFactory+ class.
def notes=(notes)
notes.each do |note|
note[:id] = nil
......
app/services/groups/import_export/export_service.rb
View file @ 2915c8be
......@@ -37,7 +37,7 @@ module Groups
end
def tree_exporter
Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
Gitlab::ImportExport::Group::TreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
end
def file_saver
......
app/services/groups/import_export/import_service.rb
View file @ 2915c8be
......@@ -34,7 +34,7 @@ module Groups
end
def restorer
@restorer ||= Gitlab::ImportExport::GroupTreeRestorer.new(user: @current_user,
@restorer ||= Gitlab::ImportExport::Group::TreeRestorer.new(user: @current_user,
shared: @shared,
group: @group,
group_hash: nil)
......
app/services/projects/import_export/export_service.rb
View file @ 2915c8be
......@@ -54,7 +54,7 @@ module Projects
end
def project_tree_saver
Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: current_user, shared: shared, params: params)
Gitlab::ImportExport::Project::TreeSaver.new(project: project, current_user: current_user, shared: shared, params: params)
end
def uploads_saver
......
doc/development/import_project.md
View file @ 2915c8be
......@@ -81,7 +81,7 @@ The last option is to import a project using a Rails console:
sudo -u git -H bundle exec rails console RAILS_ENV=production
```
1. Create a project and run `ProjectTreeRestorer`:
1. Create a project and run `Project::TreeRestorer`:
```ruby
shared_class = Struct.new(:export_path) do
......@@ -98,7 +98,7 @@ The last option is to import a project using a Rails console:
begin
#Enable Request store
RequestStore.begin!
Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project).restore
Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project).restore
ensure
RequestStore.end!
RequestStore.clear!
......@@ -128,11 +128,11 @@ The last option is to import a project using a Rails console:
For Performance testing, we should:
- Import a quite large project, [`gitlabhq`](https://gitlab.com/gitlab-org/quality/performance-data#gitlab-performance-test-framework-data) should be a good example.
- Measure the execution time of `ProjectTreeRestorer`.
- Measure the execution time of `Project::TreeRestorer`.
- Count the number of executed SQL queries during the restore.
- Observe the number of GC cycles happening.
You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `ProjectTreeRestorer`, number of SQL queries and number of GC cycles happening.
You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `Project::TreeRestorer`, number of SQL queries and number of GC cycles happening.
You can execute the script from the `gdk/gitlab` directory like this:
......
doc/user/project/settings/import_export.md
View file @ 2915c8be
......@@ -89,7 +89,7 @@ The following items will NOT be exported:
NOTE: **Note:**
For more details on the specific data persisted in a project export, see the
[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/import_export.yml) file.
[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/project/import_export.yml) file.
## Exporting a project and its data
......
ee/lib/ee/gitlab/import_export/group_project_object_builder.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module GroupProjectObjectBuilder
extend ::Gitlab::Utils::Override
extend ActiveSupport::Concern
private
override :where_clause_for_klass
def where_clause_for_klass
return attrs_to_arel(attributes.slice('filename')).and(table[:issue_id].eq(nil)) if design?
super
end
def design?
klass == DesignManagement::Design
end
end
end
end
end
ee/lib/ee/gitlab/import_export/project/object_builder.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module Project
module ObjectBuilder
extend ::Gitlab::Utils::Override
extend ActiveSupport::Concern
private
override :where_clause_for_klass
def where_clause_for_klass
return attrs_to_arel(attributes.slice('filename')).and(table[:issue_id].eq(nil)) if design?
super
end
def design?
klass == DesignManagement::Design
end
end
end
end
end
end
ee/lib/ee/gitlab/import_export/project/relation_factory.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module Project
module RelationFactory
extend ActiveSupport::Concern
EE_OVERRIDES = {
design: 'DesignManagement::Design',
designs: 'DesignManagement::Design',
design_versions: 'DesignManagement::Version',
actions: 'DesignManagement::Action',
deploy_access_levels: 'ProtectedEnvironment::DeployAccessLevel',
unprotect_access_levels: 'ProtectedBranch::UnprotectAccessLevel'
}.freeze
EE_EXISTING_OBJECT_RELATIONS = %i[DesignManagement::Design].freeze
class_methods do
extend ::Gitlab::Utils::Override
override :overrides
def overrides
super.merge(EE_OVERRIDES)
end
override :existing_object_relations
def existing_object_relations
super + EE_EXISTING_OBJECT_RELATIONS
end
end
end
end
end
end
end
ee/lib/ee/gitlab/import_export/project_relation_factory.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module ProjectRelationFactory
extend ActiveSupport::Concern
EE_OVERRIDES = {
design: 'DesignManagement::Design',
designs: 'DesignManagement::Design',
design_versions: 'DesignManagement::Version',
actions: 'DesignManagement::Action',
deploy_access_levels: 'ProtectedEnvironment::DeployAccessLevel',
unprotect_access_levels: 'ProtectedBranch::UnprotectAccessLevel'
}.freeze
EE_EXISTING_OBJECT_RELATIONS = %i[DesignManagement::Design].freeze
class_methods do
extend ::Gitlab::Utils::Override
override :overrides
def overrides
super.merge(EE_OVERRIDES)
end
override :existing_object_relations
def existing_object_relations
super + EE_EXISTING_OBJECT_RELATIONS
end
end
end
end
end
end
ee/spec/lib/ee/gitlab/import_export/group_tree_restorer_spec.rb → ee/spec/lib/ee/gitlab/import_export/group/tree_restorer_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupTreeRestorer do
describe Gitlab::ImportExport::Group::TreeRestorer do
include ImportExport::CommonUtil
let(:user) { create(:user) }
......
ee/spec/lib/ee/gitlab/import_export/group_tree_saver_spec.rb → ee/spec/lib/ee/gitlab/import_export/group/tree_saver_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupTreeSaver do
describe Gitlab::ImportExport::Group::TreeSaver do
describe 'saves the group tree into a json object' do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
......
ee/spec/lib/ee/gitlab/import_export/project_tree_restorer_spec.rb → ee/spec/lib/ee/gitlab/import_export/project/tree_restorer_spec.rb
View file @ 2915c8be
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeRestorer do
describe Gitlab::ImportExport::Project::TreeRestorer do
include ImportExport::CommonUtil
using RSpec::Parameterized::TableSyntax
......
ee/spec/lib/ee/gitlab/import_export/project_tree_saver_spec.rb → ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeSaver do
describe Gitlab::ImportExport::Project::TreeSaver do
describe 'saves the project tree into a json object' do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
......
ee/spec/lib/gitlab/import_export/group_relation_factory_spec.rb → ee/spec/lib/gitlab/import_export/group/relation_factory_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupRelationFactory do
describe Gitlab::ImportExport::Group::RelationFactory do
let(:group) { create(:group) }
let(:members_mapper) { double('members_mapper').as_null_object }
let(:user) { create(:admin) }
......@@ -11,7 +11,7 @@ describe Gitlab::ImportExport::GroupRelationFactory do
described_class.create(relation_sym: relation_sym,
relation_hash: relation_hash,
members_mapper: members_mapper,
object_builder: Gitlab::ImportExport::GroupObjectBuilder,
object_builder: Gitlab::ImportExport::Group::ObjectBuilder,
user: user,
importable: group,
excluded_keys: excluded_keys)
......
ee/spec/lib/gitlab/import_export/group_project_object_builder_spec.rb → ee/spec/lib/gitlab/import_export/project/object_builder_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupProjectObjectBuilder do
describe Gitlab::ImportExport::Project::ObjectBuilder do
let!(:group) { create(:group, :private) }
let!(:subgroup) { create(:group, :private, parent: group) }
let!(:project) do
......
lib/gitlab/import_export.rb
View file @ 2915c8be
......@@ -43,7 +43,7 @@ module Gitlab
end
def config_file
Rails.root.join('lib/gitlab/import_export/import_export.yml')
Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
end
def version_filename
......@@ -77,7 +77,7 @@ module Gitlab
end
def group_config_file
Rails.root.join('lib/gitlab/import_export/group_import_export.yml')
Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
end
end
end
......
lib/gitlab/import_export/attribute_cleaner.rb
View file @ 2915c8be
......@@ -4,8 +4,8 @@ module Gitlab
module ImportExport
class AttributeCleaner
ALLOWED_REFERENCES = [
*ProjectRelationFactory::PROJECT_REFERENCES,
*ProjectRelationFactory::USER_REFERENCES,
*Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES,
*Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES,
'group_id',
'commit_id',
'discussion_id',
......
lib/gitlab/import_export/base/object_builder.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Base
# Base class for Group & Project Object Builders.
# This class is not intended to be used on its own but
# rather inherited from.
#
# Cache keeps 1000 entries at most, 1000 is chosen based on:
# - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
# (leave some buffer it should be less than 1M). It is afforable cost for project import.
# - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
# For example, gitlab has ~970 labels and 26 milestones.
LRU_CACHE_SIZE = 1000
class ObjectBuilder
def self.build(*args)
new(*args).find
end
def initialize(klass, attributes)
@klass = klass.ancestors.include?(Label) ? Label : klass
@attributes = attributes
if Gitlab::SafeRequestStore.active?
@lru_cache = cache_from_request_store
@cache_key = [klass, attributes]
end
end
def find
find_with_cache do
find_object || klass.create(prepare_attributes)
end
end
protected
def where_clauses
raise NotImplementedError
end
# attributes wrapped in a method to be
# adjusted in sub-class if needed
def prepare_attributes
attributes
end
private
attr_reader :klass, :attributes, :lru_cache, :cache_key
def find_with_cache
return yield unless lru_cache && cache_key
lru_cache[cache_key] ||= yield
end
def cache_from_request_store
Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
end
def find_object
klass.where(where_clause).first
end
def where_clause
where_clauses.reduce(:and)
end
def table
@table ||= klass.arel_table
end
# Returns Arel clause:
# `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
# from the given Hash of attributes.
def attrs_to_arel(attrs)
attrs.map do |key, value|
table[key].eq(value)
end.reduce(:and)
end
# Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
# if attributes has 'title key, otherwise `nil`.
def where_clause_for_title
attrs_to_arel(attributes.slice('title'))
end
# Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
# if attributes has 'description key, otherwise `nil`.
def where_clause_for_description
attrs_to_arel(attributes.slice('description'))
end
# Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
# if attributes has 'created_at key, otherwise `nil`.
def where_clause_for_created_at
attrs_to_arel(attributes.slice('created_at'))
end
end
end
end
end
lib/gitlab/import_export/base/relation_factory.rb 0 → 100644
View file @ 2915c8be
This diff is collapsed.
lib/gitlab/import_export/base_object_builder.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
# Base class for Group & Project Object Builders.
# This class is not intended to be used on its own but
# rather inherited from.
#
# Cache keeps 1000 entries at most, 1000 is chosen based on:
# - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
# (leave some buffer it should be less than 1M). It is afforable cost for project import.
# - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
# For example, gitlab has ~970 labels and 26 milestones.
LRU_CACHE_SIZE = 1000
class BaseObjectBuilder
def self.build(*args)
new(*args).find
end
def initialize(klass, attributes)
@klass = klass.ancestors.include?(Label) ? Label : klass
@attributes = attributes
if Gitlab::SafeRequestStore.active?
@lru_cache = cache_from_request_store
@cache_key = [klass, attributes]
end
end
def find
find_with_cache do
find_object || klass.create(prepare_attributes)
end
end
protected
def where_clauses
raise NotImplementedError
end
# attributes wrapped in a method to be
# adjusted in sub-class if needed
def prepare_attributes
attributes
end
private
attr_reader :klass, :attributes, :lru_cache, :cache_key
def find_with_cache
return yield unless lru_cache && cache_key
lru_cache[cache_key] ||= yield
end
def cache_from_request_store
Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
end
def find_object
klass.where(where_clause).first
end
def where_clause
where_clauses.reduce(:and)
end
def table
@table ||= klass.arel_table
end
# Returns Arel clause:
# `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
# from the given Hash of attributes.
def attrs_to_arel(attrs)
attrs.map do |key, value|
table[key].eq(value)
end.reduce(:and)
end
# Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
# if attributes has 'title key, otherwise `nil`.
def where_clause_for_title
attrs_to_arel(attributes.slice('title'))
end
# Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
# if attributes has 'description key, otherwise `nil`.
def where_clause_for_description
attrs_to_arel(attributes.slice('description'))
end
# Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
# if attributes has 'created_at key, otherwise `nil`.
def where_clause_for_created_at
attrs_to_arel(attributes.slice('created_at'))
end
end
end
end
lib/gitlab/import_export/base_relation_factory.rb deleted 100644 → 0
View file @ 0b29e5a9
This diff is collapsed.
lib/gitlab/import_export/group/object_builder.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Group
# Given a class, it finds or creates a new object at group level.
#
# Example:
# `Group::ObjectBuilder.build(Label, label_attributes)`
# finds or initializes a label with the given attributes.
class ObjectBuilder < Base::ObjectBuilder
def self.build(*args)
::Group.transaction do
super
end
end
def initialize(klass, attributes)
super
@group = @attributes['group']
update_description
end
private
attr_reader :group
# Convert description empty string to nil
# due to existing object being saved with description: nil
# Which makes object lookup to fail since nil != ''
def update_description
attributes['description'] = nil if attributes['description'] == ''
end
def where_clauses
[
where_clause_base,
where_clause_for_title,
where_clause_for_description,
where_clause_for_created_at
].compact
end
# Returns Arel clause `"{table_name}"."group_id" = {group.id}`
def where_clause_base
table[:group_id].in(group_and_ancestor_ids)
end
def group_and_ancestor_ids
group.ancestors.map(&:id) << group.id
end
end
end
end
end
lib/gitlab/import_export/group/relation_factory.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Group
class RelationFactory < Base::RelationFactory
OVERRIDES = {
labels: :group_labels,
priorities: :label_priorities,
label: :group_label,
parent: :epic
}.freeze
EXISTING_OBJECT_RELATIONS = %i[
epic
epics
milestone
milestones
label
labels
group_label
group_labels
].freeze
private
def setup_models
setup_note if @relation_name == :notes
update_group_references
end
def update_group_references
return unless self.class.existing_object_relations.include?(@relation_name)
return unless @relation_hash['group_id']
@relation_hash['group_id'] = @importable.id
end
end
end
end
end
lib/gitlab/import_export/group/tree_restorer.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Group
class TreeRestorer
attr_reader :user
attr_reader :shared
attr_reader :group
def initialize(user:, shared:, group:, group_hash:)
@path = File.join(shared.export_path, 'group.json')
@user = user
@shared = shared
@group = group
@group_hash = group_hash
end
def restore
@tree_hash = @group_hash || read_tree_hash
@group_members = @tree_hash.delete('members')
@children = @tree_hash.delete('children')
if members_mapper.map && restorer.restore
@children&.each do |group_hash|
group = create_group(group_hash: group_hash, parent_group: @group)
shared = Gitlab::ImportExport::Shared.new(group)
self.class.new(
user: @user,
shared: shared,
group: group,
group_hash: group_hash
).restore
end
end
return false if @shared.errors.any?
true
rescue => e
@shared.error(e)
false
end
private
def read_tree_hash
json = IO.read(@path)
ActiveSupport::JSON.decode(json)
rescue => e
@shared.logger.error(
group_id: @group.id,
group_name: @group.name,
message: "Import/Export error: #{e.message}"
)
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
def restorer
@relation_tree_restorer ||= RelationTreeRestorer.new(
user: @user,
shared: @shared,
importable: @group,
tree_hash: @tree_hash.except('name', 'path'),
members_mapper: members_mapper,
object_builder: object_builder,
relation_factory: relation_factory,
reader: reader
)
end
def create_group(group_hash:, parent_group:)
group_params = {
name: group_hash['name'],
path: group_hash['path'],
parent_id: parent_group&.id,
visibility_level: sub_group_visibility_level(group_hash, parent_group)
}
::Groups::CreateService.new(@user, group_params).execute
end
def sub_group_visibility_level(group_hash, parent_group)
original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
if parent_group && parent_group.visibility_level < original_visibility_level
Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
else
original_visibility_level
end
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
end
def relation_factory
Gitlab::ImportExport::Group::RelationFactory
end
def object_builder
Gitlab::ImportExport::Group::ObjectBuilder
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
)
end
end
end
end
end
lib/gitlab/import_export/group/tree_saver.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Group
class TreeSaver
attr_reader :full_path, :shared
def initialize(group:, current_user:, shared:, params: {})
@params = params
@current_user = current_user
@shared = shared
@group = group
@full_path = File.join(@shared.export_path, ImportExport.group_filename)
end
def save
group_tree = serialize(@group, reader.group_tree)
tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
true
rescue => e
@shared.error(e)
false
end
private
def serialize(group, relations_tree)
group_tree = tree_saver.serialize(group, relations_tree)
group.children.each do |child|
group_tree['children'] ||= []
group_tree['children'] << serialize(child, relations_tree)
end
group_tree
rescue => e
@shared.error(e)
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
end
lib/gitlab/import_export/group_object_builder.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
# Given a class, it finds or creates a new object at group level.
#
# Example:
# `GroupObjectBuilder.build(Label, label_attributes)`
# finds or initializes a label with the given attributes.
class GroupObjectBuilder < BaseObjectBuilder
def self.build(*args)
Group.transaction do
super
end
end
def initialize(klass, attributes)
super
@group = @attributes['group']
update_description
end
private
attr_reader :group
# Convert description empty string to nil
# due to existing object being saved with description: nil
# Which makes object lookup to fail since nil != ''
def update_description
attributes['description'] = nil if attributes['description'] == ''
end
def where_clauses
[
where_clause_base,
where_clause_for_title,
where_clause_for_description,
where_clause_for_created_at
].compact
end
# Returns Arel clause `"{table_name}"."group_id" = {group.id}`
def where_clause_base
table[:group_id].in(group_and_ancestor_ids)
end
def group_and_ancestor_ids
group.ancestors.map(&:id) << group.id
end
end
end
end
lib/gitlab/import_export/group_project_object_builder.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
# Given a class, it finds or creates a new object
# (initializes in the case of Label) at group or project level.
# If it does not exist in the group, it creates it at project level.
#
# Example:
# `GroupProjectObjectBuilder.build(Label, label_attributes)`
# finds or initializes a label with the given attributes.
#
# It also adds some logic around Group Labels/Milestones for edge cases.
class GroupProjectObjectBuilder < BaseObjectBuilder
def self.build(*args)
Project.transaction do
super
end
end
def initialize(klass, attributes)
super
@group = @attributes['group']
@project = @attributes['project']
end
def find
return if epic? && group.nil?
super
end
private
attr_reader :group, :project
def where_clauses
[
where_clause_base,
where_clause_for_title,
where_clause_for_klass
].compact
end
# Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
# For example: merge_request has :target_project_id, and we are searching by :iid
# or, if group is present:
# `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
def where_clause_base
[].tap do |clauses|
clauses << table[:project_id].eq(project.id) if project
clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
end.reduce(:or)
end
# Returns Arel clause for a particular model or `nil`.
def where_clause_for_klass
attrs_to_arel(attributes.slice('iid')) if merge_request?
end
def prepare_attributes
attributes.dup.tap do |atts|
atts.delete('group') unless epic?
if label?
atts['type'] = 'ProjectLabel' # Always create project labels
elsif milestone?
if atts['group_id'] # Transform new group milestones into project ones
atts['iid'] = nil
atts.delete('group_id')
else
claim_iid
end
end
atts['importing'] = true if klass.ancestors.include?(Importable)
end
end
def label?
klass == Label
end
def milestone?
klass == Milestone
end
def merge_request?
klass == MergeRequest
end
def epic?
klass == Epic
end
# If an existing group milestone used the IID
# claim the IID back and set the group milestone to use one available
# This is necessary to fix situations like the following:
# - Importing into a user namespace project with exported group milestones
# where the IID of the Group milestone could conflict with a project one.
def claim_iid
# The milestone has to be a group milestone, as it's the only case where
# we set the IID as the maximum. The rest of them are fixed.
milestone = project.milestones.find_by(iid: attributes['iid'])
return unless milestone
milestone.iid = nil
milestone.ensure_project_iid!
milestone.save!
end
end
end
end
Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')
lib/gitlab/import_export/group_relation_factory.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class GroupRelationFactory < BaseRelationFactory
OVERRIDES = {
labels: :group_labels,
priorities: :label_priorities,
label: :group_label,
parent: :epic
}.freeze
EXISTING_OBJECT_RELATIONS = %i[
epic
epics
milestone
milestones
label
labels
group_label
group_labels
].freeze
private
def setup_models
setup_note if @relation_name == :notes
update_group_references
end
def update_group_references
return unless self.class.existing_object_relations.include?(@relation_name)
return unless @relation_hash['group_id']
@relation_hash['group_id'] = @importable.id
end
end
end
end
lib/gitlab/import_export/group_tree_restorer.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class GroupTreeRestorer
attr_reader :user
attr_reader :shared
attr_reader :group
def initialize(user:, shared:, group:, group_hash:)
@path = File.join(shared.export_path, 'group.json')
@user = user
@shared = shared
@group = group
@group_hash = group_hash
end
def restore
@tree_hash = @group_hash || read_tree_hash
@group_members = @tree_hash.delete('members')
@children = @tree_hash.delete('children')
if members_mapper.map && restorer.restore
@children&.each do |group_hash|
group = create_group(group_hash: group_hash, parent_group: @group)
shared = Gitlab::ImportExport::Shared.new(group)
self.class.new(
user: @user,
shared: shared,
group: group,
group_hash: group_hash
).restore
end
end
return false if @shared.errors.any?
true
rescue => e
@shared.error(e)
false
end
private
def read_tree_hash
json = IO.read(@path)
ActiveSupport::JSON.decode(json)
rescue => e
@shared.logger.error(
group_id: @group.id,
group_name: @group.name,
message: "Import/Export error: #{e.message}"
)
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
def restorer
@relation_tree_restorer ||= RelationTreeRestorer.new(
user: @user,
shared: @shared,
importable: @group,
tree_hash: @tree_hash.except('name', 'path'),
members_mapper: members_mapper,
object_builder: object_builder,
relation_factory: relation_factory,
reader: reader
)
end
def create_group(group_hash:, parent_group:)
group_params = {
name: group_hash['name'],
path: group_hash['path'],
parent_id: parent_group&.id,
visibility_level: sub_group_visibility_level(group_hash, parent_group)
}
::Groups::CreateService.new(@user, group_params).execute
end
def sub_group_visibility_level(group_hash, parent_group)
original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
if parent_group && parent_group.visibility_level < original_visibility_level
Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
else
original_visibility_level
end
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
end
def relation_factory
Gitlab::ImportExport::GroupRelationFactory
end
def object_builder
Gitlab::ImportExport::GroupObjectBuilder
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
)
end
end
end
end
lib/gitlab/import_export/group_tree_saver.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class GroupTreeSaver
attr_reader :full_path, :shared
def initialize(group:, current_user:, shared:, params: {})
@params = params
@current_user = current_user
@shared = shared
@group = group
@full_path = File.join(@shared.export_path, ImportExport.group_filename)
end
def save
group_tree = serialize(@group, reader.group_tree)
tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
true
rescue => e
@shared.error(e)
false
end
private
def serialize(group, relations_tree)
group_tree = tree_saver.serialize(group, relations_tree)
group.children.each do |child|
group_tree['children'] ||= []
group_tree['children'] << serialize(child, relations_tree)
end
group_tree
rescue => e
@shared.error(e)
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
config: Gitlab::ImportExport.group_config_file
).to_h
)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
lib/gitlab/import_export/importer.rb
View file @ 2915c8be
......@@ -49,7 +49,7 @@ module Gitlab
end
def project_tree
@project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
@project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user,
shared: shared,
project: project)
end
......@@ -125,7 +125,7 @@ module Gitlab
def project_to_overwrite
strong_memoize(:project_to_overwrite) do
Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
end
end
end
......
lib/gitlab/import_export/members_mapper.rb
View file @ 2915c8be
......@@ -91,9 +91,9 @@ module Gitlab
def relation_class
case @importable
when Project
when ::Project
ProjectMember
when Group
when ::Group
GroupMember
end
end
......
lib/gitlab/import_export/project/object_builder.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
# Given a class, it finds or creates a new object
# (initializes in the case of Label) at group or project level.
# If it does not exist in the group, it creates it at project level.
#
# Example:
# `ObjectBuilder.build(Label, label_attributes)`
# finds or initializes a label with the given attributes.
#
# It also adds some logic around Group Labels/Milestones for edge cases.
class ObjectBuilder < Base::ObjectBuilder
def self.build(*args)
::Project.transaction do
super
end
end
def initialize(klass, attributes)
super
@group = @attributes['group']
@project = @attributes['project']
end
def find
return if epic? && group.nil?
super
end
private
attr_reader :group, :project
def where_clauses
[
where_clause_base,
where_clause_for_title,
where_clause_for_klass
].compact
end
# Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
# For example: merge_request has :target_project_id, and we are searching by :iid
# or, if group is present:
# `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
def where_clause_base
[].tap do |clauses|
clauses << table[:project_id].eq(project.id) if project
clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
end.reduce(:or)
end
# Returns Arel clause for a particular model or `nil`.
def where_clause_for_klass
attrs_to_arel(attributes.slice('iid')) if merge_request?
end
def prepare_attributes
attributes.dup.tap do |atts|
atts.delete('group') unless epic?
if label?
atts['type'] = 'ProjectLabel' # Always create project labels
elsif milestone?
if atts['group_id'] # Transform new group milestones into project ones
atts['iid'] = nil
atts.delete('group_id')
else
claim_iid
end
end
atts['importing'] = true if klass.ancestors.include?(Importable)
end
end
def label?
klass == Label
end
def milestone?
klass == Milestone
end
def merge_request?
klass == MergeRequest
end
def epic?
klass == Epic
end
# If an existing group milestone used the IID
# claim the IID back and set the group milestone to use one available
# This is necessary to fix situations like the following:
# - Importing into a user namespace project with exported group milestones
# where the IID of the Group milestone could conflict with a project one.
def claim_iid
# The milestone has to be a group milestone, as it's the only case where
# we set the IID as the maximum. The rest of them are fixed.
milestone = project.milestones.find_by(iid: attributes['iid'])
return unless milestone
milestone.iid = nil
milestone.ensure_project_iid!
milestone.save!
end
end
end
end
end
Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder')
lib/gitlab/import_export/project/relation_factory.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
class RelationFactory < Base::RelationFactory
prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
OVERRIDES = { snippets: :project_snippets,
ci_pipelines: 'Ci::Pipeline',
pipelines: 'Ci::Pipeline',
stages: 'Ci::Stage',
statuses: 'commit_status',
triggers: 'Ci::Trigger',
pipeline_schedules: 'Ci::PipelineSchedule',
builds: 'Ci::Build',
runners: 'Ci::Runner',
hooks: 'ProjectHook',
merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
push_access_levels: 'ProtectedBranch::PushAccessLevel',
create_access_levels: 'ProtectedTag::CreateAccessLevel',
labels: :project_labels,
priorities: :label_priorities,
auto_devops: :project_auto_devops,
label: :project_label,
custom_attributes: 'ProjectCustomAttribute',
project_badges: 'Badge',
metrics: 'MergeRequest::Metrics',
ci_cd_settings: 'ProjectCiCdSetting',
error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
links: 'Releases::Link',
metrics_setting: 'ProjectMetricsSetting' }.freeze
BUILD_MODELS = %i[Ci::Build commit_status].freeze
GROUP_REFERENCES = %w[group_id].freeze
PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
EXISTING_OBJECT_RELATIONS = %i[
milestone
milestones
label
labels
project_label
project_labels
group_label
group_labels
project_feature
merge_request
epic
ProjectCiCdSetting
container_expiration_policy
].freeze
def create
@object = super
# We preload the project, user, and group to re-use objects
@object = preload_keys(@object, PROJECT_REFERENCES, @importable)
@object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
@object = preload_keys(@object, USER_REFERENCES, @user)
end
private
def invalid_relation?
# Do not create relation if it is:
# - An unknown service
# - A legacy trigger
unknown_service? ||
(!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
end
def setup_models
case @relation_name
when :merge_request_diff_files then setup_diff
when :notes then setup_note
when :'Ci::Pipeline' then setup_pipeline
when *BUILD_MODELS then setup_build
end
update_project_references
update_group_references
end
def generate_imported_object
if @relation_name == :merge_requests
MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
else
super
end
end
def update_project_references
# If source and target are the same, populate them with the new project ID.
if @relation_hash['source_project_id']
@relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
end
@relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
end
def same_source_and_target?
@relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
end
def update_group_references
return unless existing_object?
return unless @relation_hash['group_id']
@relation_hash['group_id'] = @importable.namespace_id
end
def setup_build
@relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token')
@relation_hash.delete('commands')
@relation_hash.delete('artifacts_file_store')
@relation_hash.delete('artifacts_metadata_store')
@relation_hash.delete('artifacts_size')
end
def setup_diff
@relation_hash['diff'] = @relation_hash.delete('utf8_diff')
end
def setup_pipeline
@relation_hash.fetch('stages', []).each do |stage|
stage.statuses.each do |status|
status.pipeline = imported_object
end
end
end
def unknown_service?
@relation_name == :services && parsed_relation_hash['type'] &&
!Object.const_defined?(parsed_relation_hash['type'])
end
def legacy_trigger?
@relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
end
def preload_keys(object, references, value)
return object unless value
references.each do |key|
attribute = "#{key.delete_suffix('_id')}=".to_sym
next unless object.respond_to?(key) && object.respond_to?(attribute)
if object.read_attribute(key) == value&.id
object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
end
end
object
end
end
end
end
end
lib/gitlab/import_export/project/tree_loader.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
class TreeLoader
def load(path, dedup_entries: false)
tree_hash = ActiveSupport::JSON.decode(IO.read(path))
if dedup_entries
dedup_tree(tree_hash)
else
tree_hash
end
end
private
# This function removes duplicate entries from the given tree recursively
# by caching nodes it encounters repeatedly. We only consider nodes for
# which there can actually be multiple equivalent instances (e.g. strings,
# hashes and arrays, but not `nil`s, numbers or booleans.)
#
# The algorithm uses a recursive depth-first descent with 3 cases, starting
# with a root node (the tree/hash itself):
# - a node has already been cached; in this case we return it from the cache
# - a node has not been cached yet but should be; descend into its children
# - a node is neither cached nor qualifies for caching; this is a no-op
def dedup_tree(node, nodes_seen = {})
if nodes_seen.key?(node) && distinguishable?(node)
yield nodes_seen[node]
elsif should_dedup?(node)
nodes_seen[node] = node
case node
when Array
node.each_index do |idx|
dedup_tree(node[idx], nodes_seen) do |cached_node|
node[idx] = cached_node
end
end
when Hash
node.each do |k, v|
dedup_tree(v, nodes_seen) do |cached_node|
node[k] = cached_node
end
end
end
else
node
end
end
# We do not need to consider nodes for which there cannot be multiple instances
def should_dedup?(node)
node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
end
# We can only safely de-dup values that are distinguishable. True value objects
# are always distinguishable by nature. Hashes however can represent entities,
# which are identified by ID, not value. We therefore disallow de-duping hashes
# that do not have an `id` field, since we might risk dropping entities that
# have equal attributes yet different identities.
def distinguishable?(node)
if node.is_a?(Hash)
node.key?('id')
else
true
end
end
end
end
end
end
lib/gitlab/import_export/project/tree_restorer.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
class TreeRestorer
LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
attr_reader :user
attr_reader :shared
attr_reader :project
def initialize(user:, shared:, project:)
@user = user
@shared = shared
@project = project
@tree_loader = TreeLoader.new
end
def restore
@tree_hash = read_tree_hash
@project_members = @tree_hash.delete('project_members')
RelationRenameService.rename(@tree_hash)
if relation_tree_restorer.restore
import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@project.merge_requests.set_latest_merge_request_diff_ids!
end
true
else
false
end
rescue => e
@shared.error(e)
false
end
private
def large_project?(path)
File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
end
def read_tree_hash
path = File.join(@shared.export_path, 'project.json')
dedup_entries = large_project?(path) &&
Feature.enabled?(:dedup_project_import_metadata, project.group)
@tree_loader.load(path, dedup_entries: dedup_entries)
rescue => e
Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
def relation_tree_restorer
@relation_tree_restorer ||= RelationTreeRestorer.new(
user: @user,
shared: @shared,
importable: @project,
tree_hash: @tree_hash,
object_builder: object_builder,
members_mapper: members_mapper,
relation_factory: relation_factory,
reader: reader
)
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
importable: @project)
end
def object_builder
Project::ObjectBuilder
end
def relation_factory
Project::RelationFactory
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def import_failure_service
@import_failure_service ||= ImportFailureService.new(@project)
end
end
end
end
end
lib/gitlab/import_export/project/tree_saver.rb 0 → 100644
View file @ 2915c8be
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
class TreeSaver
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
@params = params
@project = project
@current_user = current_user
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
@shared.error(e)
false
end
private
# Aware that the resulting hash needs to be pure-hash and
# does not include any AR objects anymore, only objects that run `.to_json`
def fix_project_tree(project_tree)
if @params[:description].present?
project_tree['description'] = @params[:description]
end
project_tree['project_members'] += group_members_array
RelationRenameService.add_new_associations(project_tree)
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def group_members_array
group_members.as_json(reader.group_members_tree).each do |group_member|
group_member['source_type'] = 'Project' # Make group members project members of the future import
end
end
def group_members
return [] unless @current_user.can?(:admin_group, @project.group)
# We need `.where.not(user_id: nil)` here otherwise when a group has an
# invitee, it would make the following query return 0 rows since a NULL
# user_id would be present in the subquery
# See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
end
lib/gitlab/import_export/project_relation_factory.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class ProjectRelationFactory < BaseRelationFactory
prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
OVERRIDES = { snippets: :project_snippets,
ci_pipelines: 'Ci::Pipeline',
pipelines: 'Ci::Pipeline',
stages: 'Ci::Stage',
statuses: 'commit_status',
triggers: 'Ci::Trigger',
pipeline_schedules: 'Ci::PipelineSchedule',
builds: 'Ci::Build',
runners: 'Ci::Runner',
hooks: 'ProjectHook',
merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
push_access_levels: 'ProtectedBranch::PushAccessLevel',
create_access_levels: 'ProtectedTag::CreateAccessLevel',
labels: :project_labels,
priorities: :label_priorities,
auto_devops: :project_auto_devops,
label: :project_label,
custom_attributes: 'ProjectCustomAttribute',
project_badges: 'Badge',
metrics: 'MergeRequest::Metrics',
ci_cd_settings: 'ProjectCiCdSetting',
error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
links: 'Releases::Link',
metrics_setting: 'ProjectMetricsSetting' }.freeze
BUILD_MODELS = %i[Ci::Build commit_status].freeze
GROUP_REFERENCES = %w[group_id].freeze
PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
EXISTING_OBJECT_RELATIONS = %i[
milestone
milestones
label
labels
project_label
project_labels
group_label
group_labels
project_feature
merge_request
epic
ProjectCiCdSetting
container_expiration_policy
].freeze
def create
@object = super
# We preload the project, user, and group to re-use objects
@object = preload_keys(@object, PROJECT_REFERENCES, @importable)
@object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
@object = preload_keys(@object, USER_REFERENCES, @user)
end
private
def invalid_relation?
# Do not create relation if it is:
# - An unknown service
# - A legacy trigger
unknown_service? ||
(!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
end
def setup_models
case @relation_name
when :merge_request_diff_files then setup_diff
when :notes then setup_note
when :'Ci::Pipeline' then setup_pipeline
when *BUILD_MODELS then setup_build
end
update_project_references
update_group_references
end
def generate_imported_object
if @relation_name == :merge_requests
MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
else
super
end
end
def update_project_references
# If source and target are the same, populate them with the new project ID.
if @relation_hash['source_project_id']
@relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
end
@relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
end
def same_source_and_target?
@relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
end
def update_group_references
return unless existing_object?
return unless @relation_hash['group_id']
@relation_hash['group_id'] = @importable.namespace_id
end
def setup_build
@relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token')
@relation_hash.delete('commands')
@relation_hash.delete('artifacts_file_store')
@relation_hash.delete('artifacts_metadata_store')
@relation_hash.delete('artifacts_size')
end
def setup_diff
@relation_hash['diff'] = @relation_hash.delete('utf8_diff')
end
def setup_pipeline
@relation_hash.fetch('stages', []).each do |stage|
stage.statuses.each do |status|
status.pipeline = imported_object
end
end
end
def unknown_service?
@relation_name == :services && parsed_relation_hash['type'] &&
!Object.const_defined?(parsed_relation_hash['type'])
end
def legacy_trigger?
@relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
end
def preload_keys(object, references, value)
return object unless value
references.each do |key|
attribute = "#{key.delete_suffix('_id')}=".to_sym
next unless object.respond_to?(key) && object.respond_to?(attribute)
if object.read_attribute(key) == value&.id
object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
end
end
object
end
end
end
end
lib/gitlab/import_export/project_tree_loader.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class ProjectTreeLoader
def load(path, dedup_entries: false)
tree_hash = ActiveSupport::JSON.decode(IO.read(path))
if dedup_entries
dedup_tree(tree_hash)
else
tree_hash
end
end
private
# This function removes duplicate entries from the given tree recursively
# by caching nodes it encounters repeatedly. We only consider nodes for
# which there can actually be multiple equivalent instances (e.g. strings,
# hashes and arrays, but not `nil`s, numbers or booleans.)
#
# The algorithm uses a recursive depth-first descent with 3 cases, starting
# with a root node (the tree/hash itself):
# - a node has already been cached; in this case we return it from the cache
# - a node has not been cached yet but should be; descend into its children
# - a node is neither cached nor qualifies for caching; this is a no-op
def dedup_tree(node, nodes_seen = {})
if nodes_seen.key?(node) && distinguishable?(node)
yield nodes_seen[node]
elsif should_dedup?(node)
nodes_seen[node] = node
case node
when Array
node.each_index do |idx|
dedup_tree(node[idx], nodes_seen) do |cached_node|
node[idx] = cached_node
end
end
when Hash
node.each do |k, v|
dedup_tree(v, nodes_seen) do |cached_node|
node[k] = cached_node
end
end
end
else
node
end
end
# We do not need to consider nodes for which there cannot be multiple instances
def should_dedup?(node)
node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
end
# We can only safely de-dup values that are distinguishable. True value objects
# are always distinguishable by nature. Hashes however can represent entities,
# which are identified by ID, not value. We therefore disallow de-duping hashes
# that do not have an `id` field, since we might risk dropping entities that
# have equal attributes yet different identities.
def distinguishable?(node)
if node.is_a?(Hash)
node.key?('id')
else
true
end
end
end
end
end
lib/gitlab/import_export/project_tree_restorer.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class ProjectTreeRestorer
LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
attr_reader :user
attr_reader :shared
attr_reader :project
def initialize(user:, shared:, project:)
@user = user
@shared = shared
@project = project
@tree_loader = ProjectTreeLoader.new
end
def restore
@tree_hash = read_tree_hash
@project_members = @tree_hash.delete('project_members')
RelationRenameService.rename(@tree_hash)
if relation_tree_restorer.restore
import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@project.merge_requests.set_latest_merge_request_diff_ids!
end
true
else
false
end
rescue => e
@shared.error(e)
false
end
private
def large_project?(path)
File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
end
def read_tree_hash
path = File.join(@shared.export_path, 'project.json')
dedup_entries = large_project?(path) &&
Feature.enabled?(:dedup_project_import_metadata, project.group)
@tree_loader.load(path, dedup_entries: dedup_entries)
rescue => e
Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
end
def relation_tree_restorer
@relation_tree_restorer ||= RelationTreeRestorer.new(
user: @user,
shared: @shared,
importable: @project,
tree_hash: @tree_hash,
object_builder: object_builder,
members_mapper: members_mapper,
relation_factory: relation_factory,
reader: reader
)
end
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
importable: @project)
end
def object_builder
Gitlab::ImportExport::GroupProjectObjectBuilder
end
def relation_factory
Gitlab::ImportExport::ProjectRelationFactory
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def import_failure_service
@import_failure_service ||= ImportFailureService.new(@project)
end
end
end
end
lib/gitlab/import_export/project_tree_saver.rb deleted 100644 → 0
View file @ 0b29e5a9
# frozen_string_literal: true
module Gitlab
module ImportExport
class ProjectTreeSaver
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
@params = params
@project = project
@current_user = current_user
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
@shared.error(e)
false
end
private
# Aware that the resulting hash needs to be pure-hash and
# does not include any AR objects anymore, only objects that run `.to_json`
def fix_project_tree(project_tree)
if @params[:description].present?
project_tree['description'] = @params[:description]
end
project_tree['project_members'] += group_members_array
RelationRenameService.add_new_associations(project_tree)
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def group_members_array
group_members.as_json(reader.group_members_tree).each do |group_member|
group_member['source_type'] = 'Project' # Make group members project members of the future import
end
end
def group_members
return [] unless @current_user.can?(:admin_group, @project.group)
# We need `.where.not(user_id: nil)` here otherwise when a group has an
# invitee, it would make the following query return 0 rows since a NULL
# user_id would be present in the subquery
# See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
end
end
end
end
lib/gitlab/import_export/relation_tree_restorer.rb
View file @ 2915c8be
......@@ -69,7 +69,7 @@ module Gitlab
def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, data_hash)
return unless relation_object
return if importable_class == Project && group_model?(relation_object)
return if importable_class == ::Project && group_model?(relation_object)
relation_object.assign_attributes(importable_class_sym => @importable)
......@@ -110,7 +110,7 @@ module Gitlab
excluded_keys: excluded_keys_for_relation(importable_class_sym))
@importable.assign_attributes(params)
@importable.drop_visibility_level! if importable_class == Project
@importable.drop_visibility_level! if importable_class == ::Project
Gitlab::Timeless.timeless(@importable) do
@importable.save!
......
spec/fixtures/trace/sample_trace
View file @ 2915c8be
......@@ -1442,7 +1442,7 @@ TodoService
marks a single todo id as done
caches the number of todos of a user
Gitlab::ImportExport::ProjectTreeSaver
Gitlab::ImportExport::Project::TreeSaver
saves the project tree into a json object
saves project successfully
JSON
......
spec/lib/gitlab/import_export/base_object_builder_spec.rb → spec/lib/gitlab/import_export/base/object_builder_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::BaseObjectBuilder do
describe Gitlab::ImportExport::Base::ObjectBuilder do
let(:project) do
create(:project, :repository,
:builds_disabled,
......@@ -11,7 +11,7 @@ describe Gitlab::ImportExport::BaseObjectBuilder do
path: 'project')
end
let(:klass) { Milestone }
let(:attributes) { { 'title' => 'Test BaseObjectBuilder Milestone', 'project' => project } }
let(:attributes) { { 'title' => 'Test Base::ObjectBuilder Milestone', 'project' => project } }
subject { described_class.build(klass, attributes) }
......
spec/lib/gitlab/import_export/base_relation_factory_spec.rb → spec/lib/gitlab/import_export/base/relation_factory_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::BaseRelationFactory do
describe Gitlab::ImportExport::Base::RelationFactory do
let(:user) { create(:admin) }
let(:project) { create(:project) }
let(:members_mapper) { double('members_mapper').as_null_object }
......@@ -13,7 +13,7 @@ describe Gitlab::ImportExport::BaseRelationFactory do
subject do
described_class.create(relation_sym: relation_sym,
relation_hash: relation_hash,
object_builder: Gitlab::ImportExport::GroupProjectObjectBuilder,
object_builder: Gitlab::ImportExport::Project::ObjectBuilder,
members_mapper: members_mapper,
user: user,
importable: project,
......
spec/lib/gitlab/import_export/fork_spec.rb
View file @ 2915c8be
......@@ -24,11 +24,11 @@ describe 'forked project import' do
end
let(:saver) do
Gitlab::ImportExport::ProjectTreeSaver.new(project: project_with_repo, current_user: user, shared: shared)
Gitlab::ImportExport::Project::TreeSaver.new(project: project_with_repo, current_user: user, shared: shared)
end
let(:restorer) do
Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project)
Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project)
end
before do
......
spec/lib/gitlab/import_export/group_object_builder_spec.rb → spec/lib/gitlab/import_export/group/object_builder_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupObjectBuilder do
describe Gitlab::ImportExport::Group::ObjectBuilder do
let(:group) { create(:group) }
let(:base_attributes) do
{
......
spec/lib/gitlab/import_export/group_relation_factory_spec.rb → spec/lib/gitlab/import_export/group/relation_factory_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupRelationFactory do
describe Gitlab::ImportExport::Group::RelationFactory do
let(:group) { create(:group) }
let(:members_mapper) { double('members_mapper').as_null_object }
let(:user) { create(:admin) }
......@@ -11,7 +11,7 @@ describe Gitlab::ImportExport::GroupRelationFactory do
described_class.create(relation_sym: relation_sym,
relation_hash: relation_hash,
members_mapper: members_mapper,
object_builder: Gitlab::ImportExport::GroupObjectBuilder,
object_builder: Gitlab::ImportExport::Group::ObjectBuilder,
user: user,
importable: group,
excluded_keys: excluded_keys)
......
spec/lib/gitlab/import_export/group_tree_restorer_spec.rb → spec/lib/gitlab/import_export/group/tree_restorer_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupTreeRestorer do
describe Gitlab::ImportExport::Group::TreeRestorer do
include ImportExport::CommonUtil
let(:shared) { Gitlab::ImportExport::Shared.new(group) }
......
spec/lib/gitlab/import_export/group_tree_saver_spec.rb → spec/lib/gitlab/import_export/group/tree_saver_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupTreeSaver do
describe Gitlab::ImportExport::Group::TreeSaver do
describe 'saves the group tree into a json object' do
let(:shared) { Gitlab::ImportExport::Shared.new(group) }
let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
......@@ -72,7 +72,7 @@ describe Gitlab::ImportExport::GroupTreeSaver do
# except:
# context 'with description override' do
# context 'group members' do
# ^ These are specific for the groupTreeSaver
# ^ These are specific for the Group::TreeSaver
context 'JSON' do
let(:saved_group_json) do
group_tree_saver.save
......
spec/lib/gitlab/import_export/importer_spec.rb
View file @ 2915c8be
......@@ -63,7 +63,7 @@ describe Gitlab::ImportExport::Importer do
end
it 'restores the ProjectTree' do
expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
expect(Gitlab::ImportExport::Project::TreeRestorer).to receive(:new).and_call_original
importer.execute
end
......
spec/lib/gitlab/import_export/group_project_object_builder_spec.rb → spec/lib/gitlab/import_export/project/object_builder_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::GroupProjectObjectBuilder do
describe Gitlab::ImportExport::Project::ObjectBuilder do
let!(:group) { create(:group, :private) }
let!(:subgroup) { create(:group, :private, parent: group) }
let!(:project) do
......
spec/lib/gitlab/import_export/project_relation_factory_spec.rb → spec/lib/gitlab/import_export/project/relation_factory_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::ProjectRelationFactory do
describe Gitlab::ImportExport::Project::RelationFactory do
let(:group) { create(:group) }
let(:project) { create(:project, :repository, group: group) }
let(:members_mapper) { double('members_mapper').as_null_object }
......@@ -11,7 +11,7 @@ describe Gitlab::ImportExport::ProjectRelationFactory do
let(:created_object) do
described_class.create(relation_sym: relation_sym,
relation_hash: relation_hash,
object_builder: Gitlab::ImportExport::GroupProjectObjectBuilder,
object_builder: Gitlab::ImportExport::Project::ObjectBuilder,
members_mapper: members_mapper,
user: user,
importable: project,
......@@ -243,11 +243,11 @@ describe Gitlab::ImportExport::ProjectRelationFactory do
context 'Project references' do
let(:relation_sym) { :project_foo_model }
let(:relation_hash) do
Gitlab::ImportExport::ProjectRelationFactory::PROJECT_REFERENCES.map { |ref| { ref => 99 } }.inject(:merge)
Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES.map { |ref| { ref => 99 } }.inject(:merge)
end
class ProjectFooModel < FooModel
attr_accessor(*Gitlab::ImportExport::ProjectRelationFactory::PROJECT_REFERENCES)
attr_accessor(*Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES)
end
before do
......
spec/lib/gitlab/import_export/project_tree_loader_spec.rb → spec/lib/gitlab/import_export/project/tree_loader_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeLoader do
describe Gitlab::ImportExport::Project::TreeLoader do
let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/with_duplicates.json' }
let(:project_tree) { JSON.parse(File.read(fixture)) }
......
spec/lib/gitlab/import_export/project_tree_restorer_spec.rb → spec/lib/gitlab/import_export/project/tree_restorer_spec.rb
View file @ 2915c8be
# frozen_string_literal: true
require 'spec_helper'
include ImportExport::CommonUtil
describe Gitlab::ImportExport::ProjectTreeRestorer do
describe Gitlab::ImportExport::Project::TreeRestorer do
include ImportExport::CommonUtil
let(:shared) { project.import_export_shared }
......
spec/lib/gitlab/import_export/project_tree_saver_spec.rb → spec/lib/gitlab/import_export/project/tree_saver_spec.rb
View file @ 2915c8be
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeSaver do
describe Gitlab::ImportExport::Project::TreeSaver do
describe 'saves the project tree into a json object' do
let(:shared) { project.import_export_shared }
let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) }
......@@ -75,7 +75,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
# except:
# context 'with description override' do
# context 'group members' do
# ^ These are specific for the ProjectTreeSaver
# ^ These are specific for the Project::TreeSaver
context 'JSON' do
let(:saved_project_json) do
project_tree_saver.save
......
spec/lib/gitlab/import_export/relation_rename_service_spec.rb
View file @ 2915c8be
......@@ -22,7 +22,7 @@ describe Gitlab::ImportExport::RelationRenameService do
end
context 'when importing' do
let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) }
let(:project_tree_restorer) { Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) }
let(:file_content) { IO.read(File.join(shared.export_path, 'project.json')) }
let(:json_file) { ActiveSupport::JSON.decode(file_content) }
......@@ -99,7 +99,7 @@ describe Gitlab::ImportExport::RelationRenameService do
let(:relation_tree_saver) { Gitlab::ImportExport::RelationTreeSaver.new }
let(:project_tree_saver) do
Gitlab::ImportExport::ProjectTreeSaver.new(
Gitlab::ImportExport::Project::TreeSaver.new(
project: project, current_user: user, shared: shared)
end
......
spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb
View file @ 2915c8be
......@@ -39,8 +39,8 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
context 'when restoring a project' do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') }
let(:object_builder) { Gitlab::ImportExport::GroupProjectObjectBuilder }
let(:relation_factory) { Gitlab::ImportExport::ProjectRelationFactory }
let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder }
let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory }
let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) }
let(:tree_hash) { importable_hash }
......
spec/services/groups/import_export/export_service_spec.rb
View file @ 2915c8be
......@@ -19,7 +19,7 @@ describe Groups::ImportExport::ExportService do
end
it 'saves the models' do
expect(Gitlab::ImportExport::GroupTreeSaver).to receive(:new).and_call_original
expect(Gitlab::ImportExport::Group::TreeSaver).to receive(:new).and_call_original
service.execute
end
......
spec/services/projects/import_export/export_service_spec.rb
View file @ 2915c8be
......@@ -27,7 +27,7 @@ describe Projects::ImportExport::ExportService do
end
it 'saves the models' do
expect(Gitlab::ImportExport::ProjectTreeSaver).to receive(:new).and_call_original
expect(Gitlab::ImportExport::Project::TreeSaver).to receive(:new).and_call_original
service.execute
end
......
spec/support/import_export/common_util.rb
View file @ 2915c8be
......@@ -34,13 +34,13 @@ module ImportExport
end
def get_project_restorer(project, import_path)
Gitlab::ImportExport::ProjectTreeRestorer.new(
Gitlab::ImportExport::Project::TreeRestorer.new(
user: project.creator, shared: get_shared_env(path: import_path), project: project
)
end
def get_project_saver(project, export_path)
Gitlab::ImportExport::ProjectTreeSaver.new(
Gitlab::ImportExport::Project::TreeSaver.new(
project: project, current_user: project.creator, shared: get_shared_env(path: export_path)
)
end
......
spec/support/import_export/configuration_helper.rb
View file @ 2915c8be
......@@ -36,8 +36,8 @@ module ConfigurationHelper
end
def relation_class_for_name(relation_name)
relation_name = Gitlab::ImportExport::ProjectRelationFactory.overrides[relation_name.to_sym] || relation_name
Gitlab::ImportExport::ProjectRelationFactory.relation_class(relation_name)
relation_name = Gitlab::ImportExport::Project::RelationFactory.overrides[relation_name.to_sym] || relation_name
Gitlab::ImportExport::Project::RelationFactory.relation_class(relation_name)
end
def parsed_attributes(relation_name, attributes, config: Gitlab::ImportExport.config_file)
......
spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb
View file @ 2915c8be
# frozen_string_literal: true
# Shared examples for ProjectTreeRestorer (shared to allow the testing
# Shared examples for Project::TreeRestorer (shared to allow the testing
# of EE-specific features)
RSpec.shared_examples 'restores project successfully' do |**results|
it 'restores the project' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7