Merge branch 'kassio/bulkimport-entity-trackers' into 'master'

Add BulkImports::Tracker

See merge request gitlab-org/gitlab!47009

app/models/bulk_imports/entity.rb

@@ -26,6 +26,10 @@ class BulkImports::Entity < ApplicationRecord
   belongs_to :project, optional: true
   belongs_to :group, foreign_key: :namespace_id, optional: true
 
+  has_many :trackers,
+    class_name: 'BulkImports::Tracker',
+    foreign_key: :bulk_import_entity_id
+
   validates :project, absence: true, if: :group
   validates :group, absence: true, if: :project
   validates :source_type, :source_full_path, :destination_name,

app/models/bulk_imports/tracker.rb

+# frozen_string_literal: true
+
+# This model is responsible for keeping track of the requests/pagination
+# happening during a Group Migration (BulkImport).
+class BulkImports::Tracker < ApplicationRecord
+  self.table_name = 'bulk_import_trackers'
+
+  belongs_to :entity,
+    class_name: 'BulkImports::Entity',
+    foreign_key: :bulk_import_entity_id,
+    optional: false
+
+  validates :relation,
+    presence: true,
+    uniqueness: { scope: :bulk_import_entity_id }
+
+  validates :next_page, presence: { if: :has_next_page? }
+end

changelogs/unreleased/kassio-bulkimport-entity-trackers.yml

+---
+title: Add BulkImport::Tracker to store the pagination information of the Group Migration
+  (BulkImport) requests
+merge_request: 47009
+author:
+type: changed

db/migrate/20201104204739_create_bulk_import_trackers.rb

+# frozen_string_literal: true
+
+class CreateBulkImportTrackers < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      unless table_exists?(:bulk_import_trackers)
+        create_table :bulk_import_trackers do |t|
+          t.references :bulk_import_entity,
+            null: false,
+            index: false,
+            foreign_key: { on_delete: :cascade }
+
+          t.text :relation, null: false
+          t.text :next_page
+          t.boolean :has_next_page, default: false, null: false
+
+          t.index %w(bulk_import_entity_id relation),
+            unique: true,
+            name: :bulk_import_trackers_uniq_relation_by_entity
+        end
+      end
+    end
+
+    add_check_constraint :bulk_import_trackers,
+      '(has_next_page IS FALSE or next_page IS NOT NULL)',
+      'check_next_page_requirement'
+    add_text_limit :bulk_import_trackers, :relation, 255
+    add_text_limit :bulk_import_trackers, :next_page, 255
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :bulk_import_trackers
+    end
+  end
+end

db/schema_migrations/20201104204739

+9431c771b14d61851e8e69b3a789f222463bbe460078a35c8ad3cbcf8df8b077
\ No newline at end of file

db/structure.sql

@@ -9918,6 +9918,26 @@ CREATE SEQUENCE bulk_import_entities_id_seq
 
 ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id;
 
+CREATE TABLE bulk_import_trackers (
+    id bigint NOT NULL,
+    bulk_import_entity_id bigint NOT NULL,
+    relation text NOT NULL,
+    next_page text,
+    has_next_page boolean DEFAULT false NOT NULL,
+    CONSTRAINT check_2d45cae629 CHECK ((char_length(relation) <= 255)),
+    CONSTRAINT check_40aeaa600b CHECK ((char_length(next_page) <= 255)),
+    CONSTRAINT check_next_page_requirement CHECK (((has_next_page IS FALSE) OR (next_page IS NOT NULL)))
+);
+
+CREATE SEQUENCE bulk_import_trackers_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE bulk_import_trackers_id_seq OWNED BY bulk_import_trackers.id;
+
 CREATE TABLE bulk_imports (
     id bigint NOT NULL,
     user_id integer NOT NULL,
@@ -17621,6 +17641,8 @@ ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval(
 
 ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass);
 
+ALTER TABLE ONLY bulk_import_trackers ALTER COLUMN id SET DEFAULT nextval('bulk_import_trackers_id_seq'::regclass);
+
 ALTER TABLE ONLY bulk_imports ALTER COLUMN id SET DEFAULT nextval('bulk_imports_id_seq'::regclass);
 
 ALTER TABLE ONLY chat_names ALTER COLUMN id SET DEFAULT nextval('chat_names_id_seq'::regclass);
@@ -18631,6 +18653,9 @@ ALTER TABLE ONLY bulk_import_configurations
 ALTER TABLE ONLY bulk_import_entities
     ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY bulk_import_trackers
+    ADD CONSTRAINT bulk_import_trackers_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY bulk_imports
     ADD CONSTRAINT bulk_imports_pkey PRIMARY KEY (id);
 
@@ -19914,6 +19939,8 @@ CREATE INDEX backup_labels_title_idx ON backup_labels USING btree (title);
 
 CREATE INDEX backup_labels_type_project_id_idx ON backup_labels USING btree (type, project_id);
 
+CREATE UNIQUE INDEX bulk_import_trackers_uniq_relation_by_entity ON bulk_import_trackers USING btree (bulk_import_entity_id, relation);
+
 CREATE INDEX ci_builds_gitlab_monitor_metrics ON ci_builds USING btree (status, created_at, project_id) WHERE ((type)::text = 'Ci::Build'::text);
 
 CREATE INDEX code_owner_approval_required ON protected_branches USING btree (project_id, code_owner_approval_required) WHERE (code_owner_approval_required = true);
@@ -24122,6 +24149,9 @@ ALTER TABLE ONLY analytics_cycle_analytics_group_stages
 ALTER TABLE ONLY metrics_dashboard_annotations
     ADD CONSTRAINT fk_rails_aeb11a7643 FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY bulk_import_trackers
+    ADD CONSTRAINT fk_rails_aed566d3f3 FOREIGN KEY (bulk_import_entity_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY pool_repositories
     ADD CONSTRAINT fk_rails_af3f8c5d62 FOREIGN KEY (shard_id) REFERENCES shards(id) ON DELETE RESTRICT;
 

spec/factories/bulk_import/trackers.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :bulk_import_tracker, class: 'BulkImports::Tracker' do
+    association :entity, factory: :bulk_import_entity
+
+    relation { :relation }
+    has_next_page { false }
+  end
+end

spec/models/bulk_imports/tracker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Tracker, type: :model do
+  describe 'associations' do
+    it { is_expected.to belong_to(:entity).required }
+  end
+
+  describe 'validations' do
+    before do
+      create(:bulk_import_tracker)
+    end
+
+    it { is_expected.to validate_presence_of(:relation) }
+    it { is_expected.to validate_uniqueness_of(:relation).scoped_to(:bulk_import_entity_id) }
+
+    context 'when has_next_page is true' do
+      it "validates presence of `next_page`" do
+        tracker = build(:bulk_import_tracker, has_next_page: true)
+
+        expect(tracker).not_to be_valid
+        expect(tracker.errors).to include(:next_page)
+      end
+    end
+  end
+end