Merge branch 'reschedule-issue-tracker-migration' into 'master'

Migrate issue tracker data to data field tables

See merge request gitlab-org/gitlab!24076

changelogs/unreleased/reschedule-issue-tracker-migration.yml

+---
+title: Migrate issue tracker data to data field tables
+merge_request: 24076
+author:
+type: other

db/post_migrate/20200130145430_reschedule_migrate_issue_trackers_data.rb

+# frozen_string_literal: true
+
+class RescheduleMigrateIssueTrackersData < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 3.minutes.to_i
+  BATCH_SIZE = 5_000
+  MIGRATION = 'MigrateIssueTrackersSensitiveData'
+
+  disable_ddl_transaction!
+
+  class Service < ActiveRecord::Base
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled
+
+    include ::EachBatch
+  end
+
+  def up
+    relation = Service.where(category: 'issue_tracker').where("properties IS NOT NULL AND properties != '{}' AND properties != ''")
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_issue_tracker_data_sql = "DELETE FROM issue_tracker_data WHERE \
+        (length(encrypted_issues_url) > 0 AND encrypted_issues_url_iv IS NULL) \
+        OR (length(encrypted_new_issue_url) > 0 AND encrypted_new_issue_url_iv IS NULL) \
+        OR (length(encrypted_project_url) > 0 AND encrypted_project_url_iv IS NULL)"
+
+    execute(remove_issue_tracker_data_sql)
+
+    remove_jira_tracker_data_sql = "DELETE FROM jira_tracker_data WHERE \
+        (length(encrypted_api_url) > 0 AND encrypted_api_url_iv IS NULL) \
+        OR (length(encrypted_url) > 0 AND encrypted_url_iv IS NULL) \
+        OR (length(encrypted_username) > 0 AND encrypted_username_iv IS NULL) \
+        OR (length(encrypted_password) > 0 AND encrypted_password_iv IS NULL)"
+
+    execute(remove_jira_tracker_data_sql)
+  end
+end

spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb

 # frozen_string_literal: true
 require 'spec_helper'
 
-describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, :migration, schema: 20190924152703 do
+describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, :migration, schema: 20200130145430 do
   let(:services) { table(:services) }
 
   # we need to define the classes due to encryption

spec/migrations/20200130145430_reschedule_migrate_issue_trackers_data_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200130145430_reschedule_migrate_issue_trackers_data.rb')
+
+describe RescheduleMigrateIssueTrackersData, :migration, :sidekiq do
+  let(:services) { table(:services) }
+  let(:migration_class) { Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData }
+  let(:migration_name)  { migration_class.to_s.demodulize }
+
+  let(:properties) do
+    {
+      'url' => 'http://example.com'
+    }
+  end
+  let!(:jira_service) do
+    services.create(id: 10, type: 'JiraService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:jira_service_nil) do
+    services.create(id: 11, type: 'JiraService', properties: nil, category: 'issue_tracker')
+  end
+  let!(:bugzilla_service) do
+    services.create(id: 12, type: 'BugzillaService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:youtrack_service) do
+    services.create(id: 13, type: 'YoutrackService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:youtrack_service_empty) do
+    services.create(id: 14, type: 'YoutrackService', properties: '', category: 'issue_tracker')
+  end
+  let!(:gitlab_service) do
+    services.create(id: 15, type: 'GitlabIssueTrackerService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:gitlab_service_empty) do
+    services.create(id: 16, type: 'GitlabIssueTrackerService', properties: {}, category: 'issue_tracker')
+  end
+  let!(:other_service) do
+    services.create(id: 17, type: 'OtherService', properties: properties, category: 'other_category')
+  end
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 2)
+  end
+
+  describe "#up" do
+    it 'schedules background migrations at correct time' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(migration_name).to be_scheduled_delayed_migration(3.minutes, jira_service.id, bugzilla_service.id)
+          expect(migration_name).to be_scheduled_delayed_migration(6.minutes, youtrack_service.id, gitlab_service.id)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+
+  describe "#down" do
+    let(:issue_tracker_data) { table(:issue_tracker_data) }
+    let(:jira_tracker_data) { table(:jira_tracker_data) }
+
+    let!(:valid_issue_tracker_data) do
+      issue_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_issues_url: 'http://url.com',
+        encrypted_issues_url_iv: 'somevalue'
+      )
+    end
+    let!(:invalid_issue_tracker_data) do
+      issue_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_issues_url: 'http:url.com',
+        encrypted_issues_url_iv: nil
+      )
+    end
+    let!(:valid_jira_tracker_data) do
+      jira_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_url: 'http://url.com',
+        encrypted_url_iv: 'somevalue'
+      )
+    end
+    let!(:invalid_jira_tracker_data) do
+      jira_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_url: 'http://url.com',
+        encrypted_url_iv: nil
+      )
+    end
+
+    it 'removes the invalid jira tracker data' do
+      expect { described_class.new.down }.to change { jira_tracker_data.count }.from(2).to(1)
+
+      expect(jira_tracker_data.all).to eq([valid_jira_tracker_data])
+    end
+
+    it 'removes the invalid issue tracker data' do
+      expect { described_class.new.down }.to change { issue_tracker_data.count }.from(2).to(1)
+
+      expect(issue_tracker_data.all).to eq([valid_issue_tracker_data])
+    end
+  end
+end