Adds background migrations

app/models/project.rb

@@ -651,80 +651,87 @@ class Project < ActiveRecord::Base
     import_started? || import_scheduled?
   end
 
-  def ensure_import_state(param)
-    return unless self[param] && import_state.nil?
-
-    create_import_state(status: self[:import_status],
+  def import_state_args
+    {
+      status: self[:import_status],
       jid: self[:import_jid],
-                        last_error: self[:import_error])
+      last_error: self[:import_error]
+    }
+  end
+
+  def ensure_import_state
+    return if self[:import_status] == 'none' || self[:import_status].nil?
+    return unless import_state.nil?
+
+    create_import_state(import_state_args)
 
-    update(import_status: nil)
+    update_column(:import_status, 'none')
   end
 
   def import_schedule
-    ensure_import_state(:import_status)
+    ensure_import_state
 
     import_state&.schedule
   end
 
   def force_import_start
-    ensure_import_state(:import_status)
+    ensure_import_state
 
     import_state&.force_start
   end
 
   def import_start
-    ensure_import_state(:import_status)
+    ensure_import_state
 
     import_state&.start
   end
 
   def import_fail
-    ensure_import_state(:import_status)
+    ensure_import_state
 
     import_state&.fail_op
   end
 
   def import_finish
-    ensure_import_state(:import_status)
+    ensure_import_state
 
     import_state&.finish
   end
 
   def import_jid=(new_jid)
-    return super unless import_state
+    ensure_import_state
 
-    import_state.jid = new_jid
+    import_state&.jid = new_jid
   end
 
   def import_jid
-    ensure_import_state(:import_jid)
+    ensure_import_state
 
     import_state&.jid
   end
 
   def import_error=(new_error)
-    return super unless import_state
+    ensure_import_state
 
-    import_state.last_error = new_error
+    import_state&.last_error = new_error
   end
 
   def import_error
-    ensure_import_state(:import_error)
+    ensure_import_state
 
     import_state&.last_error
   end
 
   def import_status=(new_status)
-    return super unless import_state
+    ensure_import_state
 
-    import_state.status = new_status
+    import_state&.status = new_status
   end
 
   def import_status
-    ensure_import_state(:import_status)
+    ensure_import_state
 
-    import_state&.status
+    import_state&.status || 'none'
   end
 
   def no_import?
@@ -1588,8 +1595,7 @@ class Project < ActiveRecord::Base
 
     Gitlab::SidekiqStatus.unset(import_jid)
 
-    self.import_jid = nil
-    self.save(validate: false)
+    import_state.update_column(:jid, nil)
   end
 
   def running_or_pending_build_count(force: false)
@@ -1609,8 +1615,7 @@ class Project < ActiveRecord::Base
 
     import_fail
 
-    self.import_error = sanitized_message
-    self.save(validate: false)
+    import_state.update_column(:last_error, sanitized_message)
   rescue ActiveRecord::ActiveRecordError => e
     Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
   ensure

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -64,10 +64,8 @@ module Gitlab
 
       def find_project(id)
         # TODO: Only select the JID
-        # We only care about the import JID so we can refresh it. We also only
-        # want the project if it hasn't been marked as failed yet. It's possible
-        # the import gets marked as stuck when jobs of the current stage failed
-        # somehow.
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
         Project.import_started.find_by(id: id)
       end
     end

app/workers/gitlab/github_import/refresh_import_jid_worker.rb

@@ -31,7 +31,9 @@ module Gitlab
       end
 
       def find_project(id)
-        # TODO: select only import_jid
+        # TODO: Only select the JID
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
         Project.import_started.find_by(id: id)
       end
     end

db/migrate/20180503154922_add_indexes_to_project_mirror_data.rb

+class AddIndexesToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_mirror_data, :last_successful_update_at
+    add_concurrent_index :project_mirror_data, :jid
+    add_concurrent_index :project_mirror_data, :status
+  end
+
+  def down
+    remove_index :project_mirror_data, :last_successful_update_at if index_exists? :project_mirror_data, :last_successful_update_at
+    remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
+    remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
+  end
+end

db/post_migrate/20180430144643_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb

@@ -3,6 +3,12 @@ class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord:
 
   DOWNTIME = false
 
+  UP_MIGRATION = 'PopulateImportState'.freeze
+  DOWN_MIGRATION = 'RollbackImportStateData'.freeze
+
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes
+
   disable_ddl_transaction!
 
   class Project < ActiveRecord::Base
@@ -18,54 +24,14 @@ class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord:
   end
 
   def up
-    Project.where.not(import_status: nil).each_batch do |batch|
-      start, stop = batch.pluck('MIN(id), MAX(id)').first
-
-      execute <<~SQL
-        INSERT INTO project_mirror_data (project_id, status, jid, last_update_at, last_successful_update_at, last_error)
-        SELECT id, import_status, import_jid, mirror_last_update_at, mirror_last_successful_update_at, import_error
-        FROM projects proj
-        WHERE proj.import_status IS NOT NULL
-        AND proj.id >= #{start}
-        AND proj.id < #{stop}
-      SQL
+    projects = Project.where.not(import_status: :none)
 
-      execute <<~SQL
-        UPDATE projects
-        SET import_status = NULL
-        WHERE import_status IS NOT NULL
-        AND id >= #{start}
-        AND id < #{stop}
-      SQL
-    end
+    queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
   end
 
   def down
-    ProjectImportState.where.not(status: nil).each_batch do |batch|
-      start, stop = batch.pluck('MIN(id), MAX(id)').first
-
-      execute <<~SQL
-        UPDATE projects
-        SET
-          import_status = mirror_data.status,
-          import_jid = mirror_data.jid,
-          mirror_last_update_at = mirror_data.last_update_at,
-          mirror_last_successful_update_at = mirror_data.last_successful_update_at,
-          import_error = mirror_data.last_error
-        FROM project_mirror_data mirror_data
-        WHERE mirror_data.project_id = projects.id
-        AND mirror_data.status IS NOT NULL
-        AND mirror_data.id >= #{start}
-        AND mirror_data.id < #{stop}
-      SQL
+    import_state = ProjectImportState.where.not(status: :none)
 
-      execute <<~SQL
-        UPDATE project_mirror_data
-        SET status = NULL
-        WHERE status IS NOT NULL
-        AND id >= #{start}
-        AND id < #{stop}
-      SQL
-    end
+    queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
   end
 end

db/post_migrate/20180430180136_migrate_mirror_attributes_data_from_projects_to_import_state.rb

-class MigrateMirrorAttributesDataFromProjectsToImportState < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-
-  disable_ddl_transaction!
-
-  class Project < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'projects'
-  end
-
-  def up
-    Project.joins('INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id').each_batch do |batch|
-      start, stop = batch.pluck('MIN(projects.id), MAX(projects.id)').first
-
-      execute <<~SQL
-        UPDATE project_mirror_data
-        SET
-          status = proj.import_status,
-          jid = proj.import_jid,
-          last_update_at = proj.mirror_last_update_at,
-          last_successful_update_at = proj.mirror_last_successful_update_at,
-          last_error = proj.import_error
-        FROM projects proj
-        WHERE proj.id = project_id
-        AND proj.mirror = TRUE
-        AND proj.id >= #{start}
-        AND proj.id < #{stop}
-      SQL
-
-      execute <<~SQL
-        UPDATE projects
-        SET import_status = NULL
-        WHERE mirror = TRUE
-        AND id >= #{start}
-        AND id < #{stop}
-      SQL
-    end
-  end
-
-  def down
-    Project.joins('INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id').each_batch do |batch|
-      start, stop = batch.pluck('MIN(projects.id), MAX(projects.id)').first
-
-      execute <<~SQL
-        UPDATE projects
-        SET
-          projects.import_status = import_state.status,
-          projects.import_jid = import_state.jid,
-          projects.mirror_last_update_at = import_state.last_update_at,
-          projects.mirror_last_successful_update_at = import_state.last_successful_update_at,
-          projects.import_error = import_state.last_error
-        FROM project_mirror_data import_state
-        WHERE import_state.project_id = projects.id
-        AND projects.mirror = TRUE
-        AND projects.id >= #{start}
-        AND projects.id < #{stop}
-      SQL
-
-      execute <<~SQL
-        UPDATE project_mirror_data
-        SET
-          status = NULL
-        FROM projects proj
-        WHERE proj.id = project_id
-        AND proj.mirror = TRUE
-        AND proj.id >= #{start}
-        AND proj.id < #{stop}
-      SQL
-    end
-  end
-end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180503150427) do
+ActiveRecord::Schema.define(version: 20180503154922) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1987,8 +1987,11 @@ ActiveRecord::Schema.define(version: 20180503150427) do
     t.text "last_error"
   end
 
+  add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
+  add_index "project_mirror_data", ["last_successful_update_at"], name: "index_project_mirror_data_on_last_successful_update_at", using: :btree
   add_index "project_mirror_data", ["next_execution_timestamp", "retry_count"], name: "index_mirror_data_on_next_execution_and_retry_count", using: :btree
   add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", unique: true, using: :btree
+  add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
 
   create_table "project_repository_states", force: :cascade do |t|
     t.integer "project_id", null: false
@@ -2031,6 +2034,7 @@ ActiveRecord::Schema.define(version: 20180503150427) do
     t.integer "visibility_level", default: 0, null: false
     t.boolean "archived", default: false, null: false
     t.string "avatar"
+    t.string "import_status"
     t.text "merge_requests_template"
     t.integer "star_count", default: 0, null: false
     t.boolean "merge_requests_rebase_enabled", default: false
@@ -2041,7 +2045,10 @@ ActiveRecord::Schema.define(version: 20180503150427) do
     t.boolean "merge_requests_ff_only_enabled", default: false
     t.text "issues_template"
     t.boolean "mirror", default: false, null: false
+    t.datetime "mirror_last_update_at"
+    t.datetime "mirror_last_successful_update_at"
     t.integer "mirror_user_id"
+    t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
     t.string "runners_token"
@@ -2068,6 +2075,7 @@ ActiveRecord::Schema.define(version: 20180503150427) do
     t.boolean "printing_merge_request_link_enabled", default: true, null: false
     t.integer "auto_cancel_pending_pipelines", default: 1, null: false
     t.boolean "service_desk_enabled", default: true
+    t.string "import_jid"
     t.integer "cached_markdown_version"
     t.text "delete_error"
     t.datetime "last_repository_updated_at"
@@ -2082,11 +2090,6 @@ ActiveRecord::Schema.define(version: 20180503150427) do
     t.string "external_authorization_classification_label"
     t.string "external_webhook_token"
     t.boolean "pages_https_only", default: true
-    t.string "import_status"
-    t.string "import_jid"
-    t.datetime_with_timezone "mirror_last_update_at"
-    t.datetime_with_timezone "mirror_last_successful_update_at"
-    t.text "import_error"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -2098,6 +2101,7 @@ ActiveRecord::Schema.define(version: 20180503150427) do
   add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
   add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
   add_index "projects", ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at", using: :btree
+  add_index "projects", ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at", using: :btree
   add_index "projects", ["name"], name: "index_projects_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "projects", ["namespace_id"], name: "index_projects_on_namespace_id", using: :btree
   add_index "projects", ["path"], name: "index_projects_on_path", using: :btree

ee/app/models/ee/project.rb

@@ -16,6 +16,7 @@ module EE
       before_validation :mark_remote_mirrors_for_removal
 
       before_save :set_override_pull_mirror_available, unless: -> { ::Gitlab::CurrentSettings.mirror_available }
+      before_save :set_next_execution_timestamp_to_now, if: ->(project) { project.mirror? && project.mirror_changed? && project.import_state }
 
       after_update :remove_mirror_repository_reference,
         if: ->(project) { project.mirror? && project.import_url_updated? }
@@ -134,39 +135,31 @@ module EE
       mirror? && !empty_repo?
     end
 
-    override :ensure_import_state
-    def ensure_import_state(param)
-      return unless self[param] && import_state.nil?
-
-      create_import_state(status: self[:import_status],
-                          jid: self[:import_jid],
-                          last_error: self[:import_error],
-                          last_update_at: self[:mirror_last_update_at],
+    def import_state_args
+      super.merge(last_update_at: self[:mirror_last_update_at],
                   last_successful_update_at: self[:mirror_last_successful_update_at])
-
-      update(import_status: nil)
     end
 
     def mirror_last_update_at=(new_value)
-      return super unless import_state
+      ensure_import_state
 
-      import_state.last_update_at = new_value
+      import_state&.last_update_at = new_value
     end
 
     def mirror_last_update_at
-      ensure_import_state(:mirror_last_update_at)
+      ensure_import_state
 
       import_state&.last_update_at
     end
 
     def mirror_last_successful_update_at=(new_value)
-      return super unless import_state
+      ensure_import_state
 
-      import_state.last_successful_update_at = new_value
+      import_state&.last_successful_update_at = new_value
     end
 
     def mirror_last_successful_update_at
-      ensure_import_state(:mirror_last_successful_update_at)
+      ensure_import_state
 
       import_state&.last_successful_update_at
     end
@@ -548,6 +541,10 @@ module EE
       true
     end
 
+    def set_next_execution_timestamp_to_now
+      import_state.set_next_execution_to_now
+    end
+
     def licensed_feature_available?(feature)
       available_features = strong_memoize(:licensed_feature_available) do
         Hash.new do |h, feature|

ee/app/models/ee/project_import_state.rb

@@ -89,7 +89,7 @@ module EE
     end
 
     def set_next_execution_to_now
-      return unless mirror?
+      return unless project.mirror?
 
       self.next_execution_timestamp = Time.now
     end

ee/changelogs/unreleased/44542-move-import-specific-attributes-out-of-the-project-model.yml

+---
+title: Improves database performance of mirrors, forks and imports
+merge_request: 5522
+author:
+type: performance

ee/db/post_migrate/20180430180136_migrate_mirror_attributes_data_from_projects_to_import_state.rb

+class MigrateMirrorAttributesDataFromProjectsToImportState < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+
+    scope :join_mirror_data, -> { joins('INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id') }
+  end
+
+  def up
+    Project.join_mirror_data.each_batch do |batch|
+      start, stop = batch.pluck('MIN(projects.id), MAX(projects.id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<~SQL
+          UPDATE project_mirror_data, projects
+          SET
+            project_mirror_data.status = projects.import_status,
+            project_mirror_data.jid = projects.import_jid,
+            project_mirror_data.last_update_at = projects.mirror_last_update_at,
+            project_mirror_data.last_successful_update_at = projects.mirror_last_successful_update_at,
+            project_mirror_data.last_error = projects.import_error
+          WHERE projects.id = project_mirror_data.project_id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+      else
+        execute <<~SQL
+          UPDATE project_mirror_data
+          SET
+            status = projects.import_status,
+            jid = projects.import_jid,
+            last_update_at = projects.mirror_last_update_at,
+            last_successful_update_at = projects.mirror_last_successful_update_at,
+            last_error = projects.import_error
+          FROM projects
+          WHERE projects.id = project_id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+      end
+
+      execute <<~SQL
+        UPDATE projects
+        SET import_status = 'none'
+        WHERE mirror = TRUE
+        AND id BETWEEN #{start} AND #{stop}
+      SQL
+    end
+  end
+
+  def down
+    Project.join_mirror_data.each_batch do |batch|
+      start, stop = batch.pluck('MIN(projects.id), MAX(projects.id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<~SQL
+          UPDATE projects, project_mirror_data
+          SET
+            projects.import_status = project_mirror_data.status,
+            projects.import_jid = project_mirror_data.jid,
+            projects.mirror_last_update_at = project_mirror_data.last_update_at,
+            projects.mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+            projects.import_error = project_mirror_data.last_error
+          WHERE project_mirror_data.project_id = projects.id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+
+        execute <<~SQL
+          UPDATE project_mirror_data, projects
+          SET project_mirror_data.status = 'none'
+          WHERE projects.id = project_mirror_data.project_id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+      else
+        execute <<~SQL
+          UPDATE projects
+          SET
+            import_status = project_mirror_data.status,
+            import_jid = project_mirror_data.jid,
+            mirror_last_update_at = project_mirror_data.last_update_at,
+            mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+            import_error = project_mirror_data.last_error
+          FROM project_mirror_data
+          WHERE project_mirror_data.project_id = projects.id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+
+        execute <<~SQL
+          UPDATE project_mirror_data
+          SET status = 'none'
+          FROM projects 
+          WHERE projects.id = project_id
+          AND projects.mirror = TRUE
+          AND projects.id BETWEEN #{start} AND #{stop}
+        SQL
+      end
+    end
+  end
+end

ee/spec/migrations/migrate_mirror_attributes_data_from_projects_to_import_state_spec.rb

+require 'spec_helper'
+require Rails.root.join('ee', 'db', 'post_migrate', '20180430180136_migrate_mirror_attributes_data_from_projects_to_import_state.rb')
+
+describe MigrateMirrorAttributesDataFromProjectsToImportState, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:import_state)  { table(:project_mirror_data) }
+
+  describe '#up' do
+    before do
+      namespaces.create(id: 1, name: 'gitlab-org', path: 'gitlab-org')
+
+      projects.create!(id: 1, namespace_id: 1, name: 'gitlab1',
+                       path: 'gitlab1', import_error: "foo", import_status: :started,
+                       mirror: true, import_url: generate(:url))
+      projects.create!(id: 2, namespace_id: 1, name: 'gitlab2',
+                       path: 'gitlab2', import_error: "foo", import_status: :finished,
+                       mirror: false, import_url: generate(:url))
+
+      import_state.create!(id: 1, project_id: 1)
+      import_state.create!(id: 2, project_id: 2)
+    end
+
+    it 'migrates the mirror data to the import_state table' do
+      expect(projects.joins("INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id").count).to eq(2)
+
+      expect do
+        subject.up
+      end.to change { projects.where(import_status: 'none').count }.from(0).to(1)
+
+      expect(import_state.first.status).to eq("started")
+      expect(import_state.first.last_error).to eq("foo")
+      expect(import_state.last.status).to be_nil
+      expect(import_state.last.last_error).to be_nil
+    end
+  end
+
+  describe '#down' do
+    before do
+      namespaces.create(id: 1, name: 'gitlab-org', path: 'gitlab-org')
+
+      projects.create!(id: 1, namespace_id: 1, name: 'gitlab1',
+                       path: 'gitlab1', mirror: true, import_url: generate(:url))
+
+      import_state.create!(id: 1, project_id: 1, status: :started, last_error: "foo")
+    end
+
+    it 'migrates the import_state mirror data into the projects table' do
+      expect(projects.joins("INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id").count).to eq(1)
+
+      expect do
+        subject.down
+      end.to change { import_state.where(status: 'none').count }.from(0).to(1)
+
+      expect(projects.first.import_status).to eq("started")
+      expect(projects.first.import_error).to eq("foo")
+    end
+  end
+end

ee/spec/models/project_spec.rb

@@ -48,6 +48,52 @@ describe Project do
     end
   end
 
+  describe 'setting up a mirror' do
+    context 'when new project' do
+      it 'creates import_state and sets next_execution_timestamp to now' do
+        project = build(:project, :mirror)
+
+        Timecop.freeze do
+          expect do
+            project.save
+          end.to change { ProjectImportState.count }.by(1)
+
+          expect(project.import_state.next_execution_timestamp).to eq(Time.now)
+        end
+      end
+    end
+
+    context 'when project already exists' do
+      context 'when project is not import' do
+        it 'creates import_state and sets next_execution_timestamp to now' do
+          project = create(:project)
+
+          Timecop.freeze do
+            expect do
+              project.update_attributes(mirror: true, mirror_user_id: project.creator.id, import_url: generate(:url))
+            end.to change { ProjectImportState.count }.by(1)
+
+            expect(project.import_state.next_execution_timestamp).to eq(Time.now)
+          end
+        end
+      end
+
+      context 'when project is import' do
+        it 'sets current import_state next_execution_timestamp to now' do
+          project = create(:project, import_url: generate(:url))
+
+          Timecop.freeze do
+            expect do
+              project.update_attributes(mirror: true, mirror_user_id: project.creator.id)
+            end.not_to change { ProjectImportState.count }
+
+            expect(project.import_state.next_execution_timestamp).to eq(Time.now)
+          end
+        end
+      end
+    end
+  end
+
   describe '.mirrors_to_sync' do
     let(:timestamp) { Time.now }
 

lib/gitlab/background_migration/populate_import_state.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This background migration creates all the records on the
+    # import state table for projects that are considered imports or forks
+    class PopulateImportState
+      def perform(start_id, end_id)
+        move_attributes_data_to_import_state(start_id, end_id)
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+
+      def move_attributes_data_to_import_state(start_id, end_id)
+        Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}")
+
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO project_mirror_data (project_id, status, jid, last_update_at, last_successful_update_at, last_error)
+          SELECT id, import_status, import_jid, mirror_last_update_at, mirror_last_successful_update_at, import_error
+          FROM projects
+          WHERE projects.import_status != 'none'
+          AND projects.id BETWEEN #{start_id} AND #{end_id}
+          AND NOT EXISTS (
+            SELECT id
+            FROM project_mirror_data
+            WHERE project_id = projects.id
+          )
+        SQL
+
+        ActiveRecord::Base.connection.execute <<~SQL
+          UPDATE projects
+          SET import_status = 'none'
+          WHERE import_status != 'none'
+          AND id BETWEEN #{start_id} AND #{end_id}
+        SQL
+      end
+    end
+  end
+end

lib/gitlab/background_migration/rollback_import_state_data.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This background migration migrates all the data of import_state
+    # back to the projects table for projects that are considered imports or forks
+    class RollbackImportStateData
+      def perform(start_id, end_id)
+        move_attributes_data_to_project(start_id, end_id)
+      end
+
+      def move_attributes_data_to_project(start_id, end_id)
+        Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}")
+
+        if Gitlab::Database.mysql?
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE projects, project_mirror_data
+            SET
+              projects.import_status = project_mirror_data.status,
+              projects.import_jid = project_mirror_data.jid,
+              projects.mirror_last_update_at = project_mirror_data.last_update_at,
+              projects.mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+              projects.import_error = project_mirror_data.last_error
+            WHERE project_mirror_data.project_id = projects.id
+            AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id}
+          SQL
+        else
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE projects
+            SET
+              import_status = project_mirror_data.status,
+              import_jid = project_mirror_data.jid,
+              mirror_last_update_at = project_mirror_data.last_update_at,
+              mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+              import_error = project_mirror_data.last_error
+            FROM project_mirror_data
+            WHERE project_mirror_data.project_id = projects.id
+            AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id}
+          SQL
+        end
+      end
+    end
+  end
+end

lib/gitlab/github_import/parallel_importer.rb

@@ -43,7 +43,8 @@ module Gitlab
         Gitlab::SidekiqStatus
           .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
 
-        project.update_column(:import_jid, jid)
+        project.ensure_import_state
+        project.import_state&.update_column(:jid, jid)
 
         Stage::ImportRepositoryWorker
           .perform_async(project.id)

lib/gitlab/legacy_github_import/importer.rb

@@ -78,7 +78,9 @@ module Gitlab
       def handle_errors
         return unless errors.any?
 
-        project.update_column(:import_error, {
+        project.ensure_import_state
+
+        project.import_state&.update_column(:last_error, {
           message: 'The remote data could not be fully imported.',
           errors: errors
         }.to_json)

spec/factories/projects.rb

@@ -74,7 +74,7 @@ FactoryBot.define do
       end
 
       before(:create) do |project, evaluator|
-        project.create_import_state(status: evaluator.status)
+        project.create_import_state(:status, evaluator.status)
       end
     end
 
@@ -128,7 +128,6 @@ FactoryBot.define do
         project.create_import_state(status: evaluator.status,
                                     last_update_at: evaluator.last_update_at)
       end
-
     end
 
     trait :import_hard_failed do

spec/lib/gitlab/background_migration/populate_import_state_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::PopulateImportState, :migration, schema: 20180430144643 do
+  let(:migration) { described_class.new }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:import_state)  { table(:project_mirror_data) }
+
+  before do
+    namespaces.create(id: 1, name: 'gitlab-org', path: 'gitlab-org')
+
+    projects.create!(id: 1, namespace_id: 1, name: 'gitlab1',
+                     path: 'gitlab1', import_error: "foo", import_status: :started,
+                     import_url: generate(:url))
+    projects.create!(id: 2, namespace_id: 1, name: 'gitlab2', path: 'gitlab2',
+                     import_status: :none, import_url: generate(:url))
+    projects.create!(id: 3, namespace_id: 1, name: 'gitlab3',
+                     path: 'gitlab3', import_error: "bar", import_status: :failed,
+                     import_url: generate(:url))
+
+    allow(BackgroundMigrationWorker).to receive(:perform_in)
+  end
+
+  it "creates new import_state records with project's import data" do
+    expect(projects.where.not(import_status: :none).count).to eq(2)
+
+    expect do
+      migration.perform(1, 3)
+    end.to change { import_state.all.count }.from(0).to(2)
+
+    expect(import_state.first.last_error).to eq("foo")
+    expect(import_state.last.last_error).to eq("bar")
+    expect(import_state.first.status).to eq("started")
+    expect(import_state.last.status).to eq("failed")
+    expect(projects.first.import_status).to eq("none")
+    expect(projects.last.import_status).to eq("none")
+  end
+end

spec/lib/gitlab/background_migration/rollback_import_state_data_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::RollbackImportStateData, :migration, schema: 20180430144643 do
+  let(:migration) { described_class.new }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:import_state)  { table(:project_mirror_data) }
+
+  before do
+    namespaces.create(id: 1, name: 'gitlab-org', path: 'gitlab-org')
+
+    projects.create!(id: 1, namespace_id: 1, name: 'gitlab1', import_url: generate(:url))
+    projects.create!(id: 2, namespace_id: 1, name: 'gitlab2', path: 'gitlab2', import_url: generate(:url))
+
+    import_state.create!(id: 1, project_id: 1, status: :started, last_error: "foo")
+    import_state.create!(id: 2, project_id: 2, status: :failed)
+
+    allow(BackgroundMigrationWorker).to receive(:perform_in)
+  end
+
+  it "creates new import_state records with project's import data" do
+    migration.perform(1, 2)
+
+    expect(projects.first.import_status).to eq("started")
+    expect(projects.second.import_status).to eq("failed")
+    expect(projects.first.import_error).to eq("foo")
+  end
+end

spec/lib/gitlab/github_import/parallel_importer_spec.rb

@@ -12,6 +12,8 @@ describe Gitlab::GithubImport::ParallelImporter do
     let(:importer) { described_class.new(project) }
 
     before do
+      create(:import_state, :started, project: project)
+
       expect(Gitlab::GithubImport::Stage::ImportRepositoryWorker)
         .to receive(:perform_async)
         .with(project.id)
@@ -34,8 +36,6 @@ describe Gitlab::GithubImport::ParallelImporter do
     it 'updates the import JID of the project' do
       importer.execute
 
-      debugger
-
       expect(project.reload.import_jid).to eq("github-importer/#{project.id}")
     end
   end

spec/migrations/migrate_import_attributes_data_from_projects_to_project_mirror_data_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180430144643_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb')
+
+describe MigrateImportAttributesDataFromProjectsToProjectMirrorData, :sidekiq, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:import_state) { table(:project_mirror_data) }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+    namespaces.create(id: 1, name: 'gitlab-org', path: 'gitlab-org')
+
+    projects.create!(id: 1, namespace_id: 1, name: 'gitlab1',
+                     path: 'gitlab1', import_error: "foo", import_status: :started,
+                     import_url: generate(:url))
+    projects.create!(id: 2, namespace_id: 1, name: 'gitlab2',
+                     path: 'gitlab2', import_error: "bar", import_status: :failed,
+                     import_url: generate(:url))
+    projects.create!(id: 3, namespace_id: 1, name: 'gitlab3', path: 'gitlab3', import_status: :none, import_url: generate(:url))
+  end
+
+  it 'schedules delayed background migrations in batches in bulk' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        expect(projects.where.not(import_status: :none).count).to eq(2)
+
+        subject.up
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq 2
+        expect(described_class::UP_MIGRATION).to be_scheduled_delayed_migration(5.minutes, 1, 1)
+        expect(described_class::UP_MIGRATION).to be_scheduled_delayed_migration(10.minutes, 2, 2)
+      end
+    end
+  end
+
+  describe '#down' do
+    before do
+      import_state.create!(id: 1, project_id: 1, status: :started)
+      import_state.create!(id: 2, project_id: 2, status: :started)
+    end
+
+    it 'schedules delayed background migrations in batches in bulk for rollback' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          expect(import_state.where.not(status: :none).count).to eq(2)
+
+          subject.down
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq 2
+          expect(described_class::DOWN_MIGRATION).to be_scheduled_delayed_migration(5.minutes, 1, 1)
+          expect(described_class::DOWN_MIGRATION).to be_scheduled_delayed_migration(10.minutes, 2, 2)
+        end
+      end
+    end
+  end
+end

spec/workers/gitlab/github_import/advance_stage_worker_spec.rb

@@ -107,7 +107,7 @@ describe Gitlab::GithubImport::AdvanceStageWorker, :clean_gitlab_redis_shared_st
       # This test is there to make sure we only select the columns we care
       # about.
       # TODO: enable this assertion back again
-      #expect(found.attributes).to include({ 'id' => nil, 'import_jid' => '123' })
+      # expect(found.attributes).to include({ 'id' => nil, 'import_jid' => '123' })
     end
 
     it 'returns nil if the project import is not running' do