Commit a698e7aa authored by Peter Leitzen's avatar Peter Leitzen

Merge branch 'jh-store_temp_import_data' into 'master'

[Group Migration MVC] Store temporary data used for a GitLab import

See merge request gitlab-org/gitlab!42978
parents af5d7e13 43b16a5b
# frozen_string_literal: true
class BulkImport < ApplicationRecord
belongs_to :user, optional: false
has_one :configuration, class_name: 'BulkImports::Configuration'
has_many :entities, class_name: 'BulkImports::Entity'
validates :source_type, :status, presence: true
enum source_type: { gitlab: 0 }
state_machine :status, initial: :created do
state :created, value: 0
end
end
# frozen_string_literal: true
class BulkImports::Configuration < ApplicationRecord
self.table_name = 'bulk_import_configurations'
belongs_to :bulk_import, inverse_of: :configuration, optional: false
validates :url, :access_token, length: { maximum: 255 }, presence: true
validates :url, public_url: { schemes: %w[http https], enforce_sanitization: true, ascii_only: true },
allow_nil: true
attr_encrypted :url,
key: Settings.attr_encrypted_db_key_base_truncated,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm'
attr_encrypted :access_token,
key: Settings.attr_encrypted_db_key_base_truncated,
mode: :per_attribute_iv,
algorithm: 'aes-256-gcm'
end
# frozen_string_literal: true
class BulkImports::Entity < ApplicationRecord
self.table_name = 'bulk_import_entities'
belongs_to :bulk_import, optional: false
belongs_to :parent, class_name: 'BulkImports::Entity', optional: true
belongs_to :project, optional: true
belongs_to :group, foreign_key: :namespace_id, optional: true
validates :project, absence: true, if: :group
validates :group, absence: true, if: :project
validates :source_type, :source_full_path, :destination_name,
:destination_namespace, presence: true
validate :validate_parent_is_a_group, if: :parent
validate :validate_imported_entity_type
enum source_type: { group_entity: 0, project_entity: 1 }
state_machine :status, initial: :created do
state :created, value: 0
end
private
def validate_parent_is_a_group
unless parent.group_entity?
errors.add(:parent, s_('BulkImport|must be a group'))
end
end
def validate_imported_entity_type
if group.present? && project_entity?
errors.add(:group, s_('BulkImport|expected an associated Project but has an associated Group'))
end
if project.present? && group_entity?
errors.add(:project, s_('BulkImport|expected an associated Group but has an associated Project'))
end
end
end
......@@ -167,6 +167,8 @@ class User < ApplicationRecord
has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue
has_many :assigned_merge_requests, class_name: "MergeRequest", through: :merge_request_assignees, source: :merge_request
has_many :bulk_imports
has_many :custom_attributes, class_name: 'UserCustomAttribute'
has_many :callouts, class_name: 'UserCallout'
has_many :term_agreements
......
---
title: Create a set of models to store the temporary data needed for a bulk import
merge_request: 42978
author:
type: changed
# frozen_string_literal: true
class CreateBulkImport < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
with_lock_retries do
create_table :bulk_imports do |t|
t.references :user, type: :integer, index: true, null: false, foreign_key: { on_delete: :cascade }
t.integer :source_type, null: false, limit: 2
t.integer :status, null: false, limit: 2
t.timestamps_with_timezone
end
end
end
def down
with_lock_retries do
drop_table :bulk_imports
end
end
end
# frozen_string_literal: true
class CreateBulkImportConfigurations < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
create_table :bulk_import_configurations, if_not_exists: true do |t|
t.references :bulk_import, type: :integer, index: true, null: false, foreign_key: { on_delete: :cascade }
t.text :encrypted_url # rubocop: disable Migration/AddLimitToTextColumns
t.text :encrypted_url_iv # rubocop: disable Migration/AddLimitToTextColumns
t.text :encrypted_access_token # rubocop: disable Migration/AddLimitToTextColumns
t.text :encrypted_access_token_iv # rubocop: disable Migration/AddLimitToTextColumns
t.timestamps_with_timezone
end
end
def down
drop_table :bulk_import_configurations
end
end
# frozen_string_literal: true
class CreateBulkImportEntities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
create_table :bulk_import_entities, if_not_exists: true do |t|
t.bigint :bulk_import_id, index: true, null: false
t.bigint :parent_id, index: true
t.bigint :namespace_id, index: true
t.bigint :project_id, index: true
t.integer :source_type, null: false, limit: 2
t.text :source_full_path, null: false
t.text :destination_name, null: false
t.text :destination_namespace, null: false
t.integer :status, null: false, limit: 2
t.text :jid
t.timestamps_with_timezone
end
add_text_limit(:bulk_import_entities, :source_full_path, 255)
add_text_limit(:bulk_import_entities, :destination_name, 255)
add_text_limit(:bulk_import_entities, :destination_namespace, 255)
add_text_limit(:bulk_import_entities, :jid, 255)
end
def down
drop_table :bulk_import_entities
end
end
# frozen_string_literal: true
class AddBulkImportForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :bulk_import_entities, :bulk_imports, column: :bulk_import_id, on_delete: :cascade
end
def down
remove_foreign_key :bulk_import_entities, column: :bulk_import_id
end
end
# frozen_string_literal: true
class AddParentForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :bulk_import_entities, :bulk_import_entities, column: :parent_id, on_delete: :cascade
end
def down
remove_foreign_key :bulk_import_entities, column: :parent_id
end
end
# frozen_string_literal: true
class AddNamespaceForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :bulk_import_entities, :namespaces, column: :namespace_id
end
def down
with_lock_retries do
remove_foreign_key :bulk_import_entities, column: :namespace_id
end
end
end
# frozen_string_literal: true
class AddProjectForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :bulk_import_entities, :projects, column: :project_id
end
def down
with_lock_retries do
remove_foreign_key :bulk_import_entities, column: :project_id
end
end
end
8196e28f6fe8cdb4cf710922b5cd218030ba587c629de7ee75dc061d05c7e1a9
\ No newline at end of file
a14df9e9a115d39636b29bfe73fb175bb1e8d4510bee26e3e0c6c979949b13c4
\ No newline at end of file
7d43d2fa91e27eaf9399cf0ce9e4375e849deb71b12d4891455bc51392bce14a
\ No newline at end of file
f445704e51dad2369719d8c0931c3314793fa90ba6b5a383df503ea4f6dafd20
\ No newline at end of file
a814b745b4911fc6c80971e6c0c19e6d64ca30cb94fa87a94bc1adf8c07b1c87
\ No newline at end of file
a915ccf5df0ec803286205916ffcd34b1410d1cc4da84f8299b63b3665d69e09
\ No newline at end of file
28a71a380be0ef08389defac604c351f0a7f31b6c03a7c40aabe47bf09e6a485
\ No newline at end of file
......@@ -9816,6 +9816,73 @@ CREATE SEQUENCE broadcast_messages_id_seq
ALTER SEQUENCE broadcast_messages_id_seq OWNED BY broadcast_messages.id;
CREATE TABLE bulk_import_configurations (
id bigint NOT NULL,
bulk_import_id integer NOT NULL,
encrypted_url text,
encrypted_url_iv text,
encrypted_access_token text,
encrypted_access_token_iv text,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL
);
CREATE SEQUENCE bulk_import_configurations_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE bulk_import_configurations_id_seq OWNED BY bulk_import_configurations.id;
CREATE TABLE bulk_import_entities (
id bigint NOT NULL,
bulk_import_id bigint NOT NULL,
parent_id bigint,
namespace_id bigint,
project_id bigint,
source_type smallint NOT NULL,
source_full_path text NOT NULL,
destination_name text NOT NULL,
destination_namespace text NOT NULL,
status smallint NOT NULL,
jid text,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
CONSTRAINT check_13f279f7da CHECK ((char_length(source_full_path) <= 255)),
CONSTRAINT check_715d725ea2 CHECK ((char_length(destination_name) <= 255)),
CONSTRAINT check_796a4d9cc6 CHECK ((char_length(jid) <= 255)),
CONSTRAINT check_b834fff4d9 CHECK ((char_length(destination_namespace) <= 255))
);
CREATE SEQUENCE bulk_import_entities_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id;
CREATE TABLE bulk_imports (
id bigint NOT NULL,
user_id integer NOT NULL,
source_type smallint NOT NULL,
status smallint NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL
);
CREATE SEQUENCE bulk_imports_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE bulk_imports_id_seq OWNED BY bulk_imports.id;
CREATE TABLE chat_names (
id integer NOT NULL,
user_id integer NOT NULL,
......@@ -17304,6 +17371,12 @@ ALTER TABLE ONLY boards_epic_user_preferences ALTER COLUMN id SET DEFAULT nextva
ALTER TABLE ONLY broadcast_messages ALTER COLUMN id SET DEFAULT nextval('broadcast_messages_id_seq'::regclass);
ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval('bulk_import_configurations_id_seq'::regclass);
ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass);
ALTER TABLE ONLY bulk_imports ALTER COLUMN id SET DEFAULT nextval('bulk_imports_id_seq'::regclass);
ALTER TABLE ONLY chat_names ALTER COLUMN id SET DEFAULT nextval('chat_names_id_seq'::regclass);
ALTER TABLE ONLY chat_teams ALTER COLUMN id SET DEFAULT nextval('chat_teams_id_seq'::regclass);
......@@ -18280,6 +18353,15 @@ ALTER TABLE ONLY boards
ALTER TABLE ONLY broadcast_messages
ADD CONSTRAINT broadcast_messages_pkey PRIMARY KEY (id);
ALTER TABLE ONLY bulk_import_configurations
ADD CONSTRAINT bulk_import_configurations_pkey PRIMARY KEY (id);
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id);
ALTER TABLE ONLY bulk_imports
ADD CONSTRAINT bulk_imports_pkey PRIMARY KEY (id);
ALTER TABLE ONLY chat_names
ADD CONSTRAINT chat_names_pkey PRIMARY KEY (id);
......@@ -19785,6 +19867,18 @@ CREATE INDEX index_boards_on_project_id ON boards USING btree (project_id);
CREATE INDEX index_broadcast_message_on_ends_at_and_broadcast_type_and_id ON broadcast_messages USING btree (ends_at, broadcast_type, id);
CREATE INDEX index_bulk_import_configurations_on_bulk_import_id ON bulk_import_configurations USING btree (bulk_import_id);
CREATE INDEX index_bulk_import_entities_on_bulk_import_id ON bulk_import_entities USING btree (bulk_import_id);
CREATE INDEX index_bulk_import_entities_on_namespace_id ON bulk_import_entities USING btree (namespace_id);
CREATE INDEX index_bulk_import_entities_on_parent_id ON bulk_import_entities USING btree (parent_id);
CREATE INDEX index_bulk_import_entities_on_project_id ON bulk_import_entities USING btree (project_id);
CREATE INDEX index_bulk_imports_on_user_id ON bulk_imports USING btree (user_id);
CREATE UNIQUE INDEX index_chat_names_on_service_id_and_team_id_and_chat_id ON chat_names USING btree (service_id, team_id, chat_id);
CREATE UNIQUE INDEX index_chat_names_on_user_id_and_service_id ON chat_names USING btree (user_id, service_id);
......@@ -22415,6 +22509,9 @@ ALTER TABLE ONLY ci_builds
ALTER TABLE ONLY vulnerabilities
ADD CONSTRAINT fk_88b4d546ef FOREIGN KEY (start_date_sourcing_milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT fk_88c725229f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY issues
ADD CONSTRAINT fk_899c8f3231 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
......@@ -22493,6 +22590,9 @@ ALTER TABLE ONLY ci_builds
ALTER TABLE ONLY ci_pipelines
ADD CONSTRAINT fk_a23be95014 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE;
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT fk_a44ff95be5 FOREIGN KEY (parent_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
ALTER TABLE ONLY users
ADD CONSTRAINT fk_a4b8fefe3e FOREIGN KEY (managing_group_id) REFERENCES namespaces(id) ON DELETE SET NULL;
......@@ -22535,6 +22635,9 @@ ALTER TABLE ONLY project_access_tokens
ALTER TABLE ONLY protected_tag_create_access_levels
ADD CONSTRAINT fk_b4eb82fe3c FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT fk_b69fa2b2df FOREIGN KEY (bulk_import_id) REFERENCES bulk_imports(id) ON DELETE CASCADE;
ALTER TABLE ONLY compliance_management_frameworks
ADD CONSTRAINT fk_b74c45b71f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
......@@ -22595,6 +22698,9 @@ ALTER TABLE ONLY todos
ALTER TABLE ONLY geo_event_log
ADD CONSTRAINT fk_cff7185ad2 FOREIGN KEY (reset_checksum_event_id) REFERENCES geo_reset_checksum_events(id) ON DELETE CASCADE;
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT fk_d06d023c30 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY project_mirror_data
ADD CONSTRAINT fk_d1aad367d7 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
......@@ -22835,6 +22941,9 @@ ALTER TABLE ONLY project_statistics
ALTER TABLE ONLY user_details
ADD CONSTRAINT fk_rails_12e0b3043d FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
ALTER TABLE ONLY bulk_imports
ADD CONSTRAINT fk_rails_130a09357d FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
ALTER TABLE ONLY diff_note_positions
ADD CONSTRAINT fk_rails_13c7212859 FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE;
......@@ -23147,6 +23256,9 @@ ALTER TABLE ONLY status_page_settings
ALTER TABLE ONLY project_repository_storage_moves
ADD CONSTRAINT fk_rails_5106dbd44a FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY bulk_import_configurations
ADD CONSTRAINT fk_rails_536b96bff1 FOREIGN KEY (bulk_import_id) REFERENCES bulk_imports(id) ON DELETE CASCADE;
ALTER TABLE ONLY x509_commit_signatures
ADD CONSTRAINT fk_rails_53fe41188f FOREIGN KEY (x509_certificate_id) REFERENCES x509_certificates(id) ON DELETE CASCADE;
......
......@@ -4467,6 +4467,15 @@ msgstr ""
msgid "Bulk request concurrency"
msgstr ""
msgid "BulkImport|expected an associated Group but has an associated Project"
msgstr ""
msgid "BulkImport|expected an associated Project but has an associated Group"
msgstr ""
msgid "BulkImport|must be a group"
msgstr ""
msgid "Burndown chart"
msgstr ""
......
# frozen_string_literal: true
FactoryBot.define do
factory :bulk_import, class: 'BulkImport' do
user
source_type { :gitlab }
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :bulk_import_entity, class: 'BulkImports::Entity' do
bulk_import
source_type { :group_entity }
sequence(:source_full_path) { |n| "source-path-#{n}" }
sequence(:destination_namespace) { |n| "destination-path-#{n}" }
destination_name { 'Imported Entity' }
trait(:group_entity) do
source_type { :group_entity }
end
trait(:project_entity) do
source_type { :project_entity }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImport, type: :model do
describe 'associations' do
it { is_expected.to belong_to(:user).required }
it { is_expected.to have_one(:configuration) }
it { is_expected.to have_many(:entities) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:source_type) }
it { is_expected.to validate_presence_of(:status) }
it { is_expected.to define_enum_for(:source_type).with_values(%i[gitlab]) }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Configuration, type: :model do
describe 'associations' do
it { is_expected.to belong_to(:bulk_import).required }
end
describe 'validations' do
it { is_expected.to validate_length_of(:url).is_at_most(255) }
it { is_expected.to validate_length_of(:access_token).is_at_most(255) }
it { is_expected.to validate_presence_of(:url) }
it { is_expected.to validate_presence_of(:access_token) }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Entity, type: :model do
describe 'associations' do
it { is_expected.to belong_to(:bulk_import).required }
it { is_expected.to belong_to(:parent) }
it { is_expected.to belong_to(:group) }
it { is_expected.to belong_to(:project) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:source_type) }
it { is_expected.to validate_presence_of(:source_full_path) }
it { is_expected.to validate_presence_of(:destination_name) }
it { is_expected.to validate_presence_of(:destination_namespace) }
it { is_expected.to define_enum_for(:source_type).with_values(%i[group_entity project_entity]) }
context 'when associated with a group and project' do
it 'is invalid' do
entity = build(:bulk_import_entity, group: build(:group), project: build(:project))
expect(entity).not_to be_valid
expect(entity.errors).to include(:project, :group)
end
end
context 'when not associated with a group or project' do
it 'is valid' do
entity = build(:bulk_import_entity, group: nil, project: nil)
expect(entity).to be_valid
end
end
context 'when associated with a group and no project' do
it 'is valid as a group_entity' do
entity = build(:bulk_import_entity, :group_entity, group: build(:group), project: nil)
expect(entity).to be_valid
end
it 'is invalid as a project_entity' do
entity = build(:bulk_import_entity, :project_entity, group: build(:group), project: nil)
expect(entity).not_to be_valid
expect(entity.errors).to include(:group)
end
end
context 'when associated with a project and no group' do
it 'is valid' do
entity = build(:bulk_import_entity, :project_entity, group: nil, project: build(:project))
expect(entity).to be_valid
end
it 'is invalid as a project_entity' do
entity = build(:bulk_import_entity, :group_entity, group: nil, project: build(:project))
expect(entity).not_to be_valid
expect(entity.errors).to include(:project)
end
end
context 'when the parent is a group import' do
it 'is valid' do
entity = build(:bulk_import_entity, parent: build(:bulk_import_entity, :group_entity))
expect(entity).to be_valid
end
end
context 'when the parent is a project import' do
it 'is invalid' do
entity = build(:bulk_import_entity, parent: build(:bulk_import_entity, :project_entity))
expect(entity).not_to be_valid
expect(entity.errors).to include(:parent)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment