Skip to content

G
gitlab-ce
Commit cbe6d21c authored by Andreas Brandl's avatar Andreas Brandl
Browse files
Options

Merge branch '331248-remove-old-background-migrations' into 'master' 

remove old background migrations

See merge request gitlab-org/gitlab!78184
parents 9e54019b d222483b
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1343 deletions
ee/lib/ee/gitlab/background_migration/backfill_version_data_from_gitaly.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module BackfillVersionDataFromGitaly
extend ::Gitlab::Utils::Override
class Version < ActiveRecord::Base
self.table_name = 'design_management_versions'
self.inheritance_column = :_type_disabled
# The `sha` of a version record must be deserialized from binary
# in order to convert it to a `sha` String that can be used to fetch
# a corresponding Commit from Git.
def sha
value = super
value.unpack1('H*')
end
scope :backfillable_for_issue, -> (issue_id) do
where(author_id: nil).or(where(created_at: nil))
.where(issue_id: issue_id)
end
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
self.inheritance_column = :_type_disabled
end
override :perform
def perform(issue_id)
issue = Issue.find_by_id(issue_id)
return unless issue
# We need a full Project instance in order to initialize a
# Repository instance that can perform Gitaly calls.
project = ::Project.find_by_id(issue.project_id)
return if project.nil? || project.pending_delete?
# We need a full Repository instance to perform Gitaly calls.
repository = ::DesignManagement::Repository.new(project)
versions = Version.backfillable_for_issue(issue_id)
commits = commits_for_versions(versions, repository)
ActiveRecord::Base.transaction do
versions.each do |version|
commit = commits[version.sha]
unless commit.nil?
version.update_columns(created_at: commit.created_at, author_id: commit.author&.id)
end
end
end
end
private
# Performs a Gitaly request to fetch the corresponding Commit data
# for the given versions.
#
# Returns Commits as a Hash of { sha => Commit }
def commits_for_versions(versions, repository)
shas = versions.map(&:sha)
commits = repository.commits_by(oids: shas)
# Batch load the commit authors so the `User` records are fetched
# all at once the first time we call `commit.author.id`.
commits.each(&:lazy_author)
commits.each_with_object({}) do |commit, hash|
hash[commit.id] = commit
end
end
end
end
end
end
ee/lib/ee/gitlab/background_migration/generate_gitlab_subscriptions.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module GenerateGitlabSubscriptions
extend ::Gitlab::Utils::Override
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled # Disable STI
scope :with_plan, -> { where.not(plan_id: nil) }
scope :without_subscription, -> do
joins("LEFT JOIN gitlab_subscriptions ON namespaces.id = gitlab_subscriptions.namespace_id")
.where(gitlab_subscriptions: { id: nil })
end
def trial_active?
trial_ends_on.present? && trial_ends_on >= Date.today
end
end
class GitlabSubscription < ActiveRecord::Base
self.table_name = 'gitlab_subscriptions'
end
override :perform
def perform(start_id, stop_id)
now = Time.now
# Some fields like seats or end_date will be properly updated by a script executed
# from the subscription portal after this MR hits production.
rows = Namespace
.with_plan
.without_subscription
.where(id: start_id..stop_id)
.select(:id, :plan_id, :trial_ends_on, :created_at)
.map do |namespace|
{
namespace_id: namespace.id,
hosted_plan_id: namespace.plan_id,
trial: namespace.trial_active?,
start_date: namespace.created_at.to_date,
auto_renew: false,
seats: 0,
created_at: now,
updated_at: now
}
end
ApplicationRecord.legacy_bulk_insert(:gitlab_subscriptions, rows) # rubocop:disable Gitlab/BulkInsert
end
end
end
end
end
ee/lib/ee/gitlab/background_migration/migrate_devops_segments_to_groups.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
#
module EE
module Gitlab
module BackgroundMigration
module MigrateDevopsSegmentsToGroups
class AdoptionSegmentSelection < ActiveRecord::Base
self.table_name = 'analytics_devops_adoption_segment_selections'
end
class AdoptionSegment < ActiveRecord::Base
SNAPSHOT_CALCULATION_DELAY = 60.seconds
self.table_name = 'analytics_devops_adoption_segments'
has_many :selections, class_name: 'AdoptionSegmentSelection', foreign_key: :segment_id
scope :without_namespace_id, -> { where(namespace_id: nil) }
after_commit :schedule_data_calculation, on: :create
private
def schedule_data_calculation
::Analytics::DevopsAdoption::CreateSnapshotWorker.perform_in(SNAPSHOT_CALCULATION_DELAY + rand(10), id)
end
end
def perform
ActiveRecord::Base.transaction do
AdoptionSegment
.without_namespace_id
.includes(:selections)
.sort_by { |segment| segment.selections.size }
.each do |segment|
if segment.selections.size == 1
group_id = segment.selections.first.group_id
if segment_exists?(group_id)
segment.delete
else
segment.update(namespace_id: group_id)
end
else
segment.selections.each do |selection|
unless segment_exists?(selection.group_id)
AdoptionSegment.create(namespace_id: selection.group_id)
end
end
segment.delete
end
end
end
end
private
def segment_exists?(namespace_id)
AdoptionSegment.where(namespace_id: namespace_id).exists?
end
end
end
end
end
ee/lib/ee/gitlab/background_migration/sync_blocking_issues_count.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module SyncBlockingIssuesCount
extend ::Gitlab::Utils::Override
override :perform
def perform(start_id, end_id)
ActiveRecord::Base.connection.execute <<~SQL
UPDATE issues
SET blocking_issues_count = grouped_counts.count
FROM
(
SELECT blocking_issue_id, SUM(blocked_count) AS count
FROM (
SELECT COUNT(*) AS blocked_count, issue_links.source_id AS blocking_issue_id
FROM issue_links
INNER JOIN issues ON issue_links.source_id = issues.id
WHERE issue_links.link_type = 1
AND issues.state_id = 1
AND issues.blocking_issues_count = 0
AND issue_links.source_id BETWEEN #{start_id} AND #{end_id}
GROUP BY blocking_issue_id HAVING COUNT(*) > 0
UNION ALL
SELECT COUNT(*) AS blocked_count, issue_links.target_id AS blocking_issue_id
FROM issue_links
INNER JOIN issues ON issue_links.target_id = issues.id
WHERE issue_links.link_type = 2
AND issues.state_id = 1
AND issues.blocking_issues_count = 0
AND issue_links.target_id BETWEEN #{start_id} AND #{end_id}
GROUP BY blocking_issue_id HAVING COUNT(*) > 0
) blocking_counts
GROUP BY blocking_issue_id
) AS grouped_counts
WHERE issues.blocking_issues_count = 0
AND issues.state_id = 1
AND issues.id = grouped_counts.blocking_issue_id
AND grouped_counts.count > 0
SQL
end
end
end
end
end
ee/spec/lib/ee/gitlab/background_migration/backfill_version_data_from_gitaly_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
# rubocop:disable RSpec/FactoriesInMigrationSpecs
RSpec.describe Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly do
let(:issue) { create(:issue) }
def perform_worker
described_class.new.perform(issue.id)
end
def create_version(attrs)
# Use the `:design` factory to create a version that has a
# correponding Git commit.
attrs[:issue] ||= issue
design = create(:design, :with_file, attrs)
design.versions.first
end
def create_version_with_missing_data(attrs = {})
version = create_version(attrs)
version.update_columns(author_id: nil)
version
end
it 'correctly sets version author_id and created_at properties from the Git commit' do
version = create_version_with_missing_data
commit = issue.project.design_repository.commit(version.sha)
expect(version).to have_attributes(
author_id: nil
)
expect(commit.author.id).to be_present
expect { perform_worker }.to(
change do
version.reload
version.author_id
end
.from(nil)
.to(commit.author.id)
)
end
it 'avoids N+1 issues and fetches all User records in one call' do
author_1, author_2, author_3 = create_list(:user, 3)
create_version_with_missing_data(author: author_1)
create_version_with_missing_data(author: author_2)
create_version_with_missing_data(author: author_3)
expect(User).to receive(:by_any_email).with(
array_including(author_1.email, author_2.email, author_3.email),
confirmed: true
).once.and_call_original
perform_worker
end
it 'leaves versions in a valid state' do
version = create_version_with_missing_data
expect(version).to be_valid
expect { perform_worker }.not_to change { version.reload.valid? }
end
it 'skips versions that are in projects that are pending deletion' do
version = create_version_with_missing_data
version.issue.project.update!(pending_delete: true)
expect { perform_worker }.not_to(
change do
version.reload
version.author_id
end
)
end
end
# rubocop:enable RSpec/FactoriesInMigrationSpecs
ee/spec/lib/ee/gitlab/background_migration/migrate_devops_segments_to_groups_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups, schema: 20210301200959 do
let(:segments_table) { table(:analytics_devops_adoption_segments) }
let(:selections_table) { table(:analytics_devops_adoption_segment_selections) }
let(:namespaces_table) { table(:namespaces) }
let(:namespace) { namespaces_table.create!(name: 'gitlab', path: 'gitlab-org') }
let(:namespace2) { namespaces_table.create!(name: 'gitlab-test', path: 'gitlab-test') }
let!(:single_group_segment) do
segments_table.create!.tap do |segment|
selections_table.create!(group_id: namespace.id, segment_id: segment.id)
end
end
let!(:multiple_groups_segment) do
segments_table.create!.tap do |segment|
selections_table.create!(group_id: namespace.id, segment_id: segment.id)
selections_table.create!(group_id: namespace2.id, segment_id: segment.id)
end
end
let!(:empty_segment) do
segments_table.create!
end
describe '#perform' do
it 'removes empty segments' do
expect { subject.perform }.to change { segments_table.where(id: empty_segment.id).exists? }.to(false)
end
it 'sets namespace id for segments with single group' do
expect do
subject.perform
single_group_segment.reload
end.to change { single_group_segment.namespace_id }.from(nil).to(namespace.id)
end
it 'creates segment with namespace_id for each unique group across all selections' do
expect do
subject.perform
end.to change { segments_table.where(namespace_id: [namespace.id, namespace2.id]).count }.from(0).to(2)
end
it 'schedules data calculation for fresh segments' do
expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(kind_of(Integer), kind_of(Integer))
subject.perform
end
it 'removes old multi-group segment' do
expect { subject.perform }.to change { segments_table.where(id: multiple_groups_segment.id).exists? }.to(false)
end
context 'with duplicated segments' do
let!(:single_group_segment_duplicate) do
segments_table.create!.tap do |segment|
selections_table.create!(group_id: namespace.id, segment_id: segment.id)
end
end
it 'removes duplicated segments' do
expect { subject.perform }.to change { segments_table.where(id: single_group_segment_duplicate).exists? }.to(false)
end
end
end
end
lib/gitlab/background_migration/backfill_designs_relative_position.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This migration is not needed anymore and was disabled, because we're now
# also backfilling design positions immediately before moving a design.
#
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39555
class BackfillDesignsRelativePosition
def perform(issue_ids)
# no-op
end
end
end
end
lib/gitlab/background_migration/backfill_legacy_project_repositories.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class that will fill the project_repositories table for projects that
# are on legacy storage and an entry is is missing in this table.
class BackfillLegacyProjectRepositories < BackfillProjectRepositories
private
def projects
Project.with_parent.on_legacy_storage
end
end
end
end
lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Update existent project update_at column after their repository storage was moved
class BackfillProjectUpdatedAtAfterRepositoryStorageMove
def perform(*project_ids)
updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
Project.connection.execute <<-SQL
WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{updated_repository_storages.to_sql}
)
UPDATE projects
SET updated_at = (repository_storage_cte.updated_at + interval '1 second')
FROM repository_storage_cte
WHERE projects.id = repository_storage_cte.project_id AND projects.updated_at <= repository_storage_cte.updated_at
SQL
end
end
end
end
lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class BackfillVersionDataFromGitaly
def perform(issue_id)
end
end
end
end
Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_mod_with('Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly')
lib/gitlab/background_migration/calculate_wiki_sizes.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class CalculateWikiSizes
def perform(start_id, stop_id)
::ProjectStatistics.where(wiki_size: nil)
.where(id: start_id..stop_id)
.includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics|
statistics.refresh!(only: [:wiki_size])
rescue StandardError => e
Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}"
end
end
end
end
end
lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class CleanupOptimisticLockingNulls
QUERY_ITEM_SIZE = 1_000
# table - The name of the table the migration is performed for.
# start_id - The ID of the object to start at
# stop_id - The ID of the object to end at
def perform(start_id, stop_id, table)
model = define_model_for(table)
# After analysis done, a batch size of 1,000 items per query was found to be
# the most optimal. Discussion in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18418#note_282285336
(start_id..stop_id).each_slice(QUERY_ITEM_SIZE).each do |range|
model
.where(lock_version: nil)
.where("ID BETWEEN ? AND ?", range.first, range.last)
.update_all(lock_version: 0)
end
end
def define_model_for(table)
Class.new(ActiveRecord::Base) do
self.table_name = table
end
end
end
end
end
lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# save validity time pages domain
class FillValidTimeForPagesDomainCertificate
# define PagesDomain with only needed code
class PagesDomain < ActiveRecord::Base
self.table_name = 'pages_domains'
def x509
return unless certificate.present?
@x509 ||= OpenSSL::X509::Certificate.new(certificate)
rescue OpenSSL::X509::CertificateError
nil
end
end
def perform(start_id, stop_id)
PagesDomain.where(id: start_id..stop_id).find_each do |domain|
# for some reason activerecord doesn't append timezone, iso8601 forces this
domain.update_columns(
certificate_valid_not_before: domain.x509&.not_before&.iso8601,
certificate_valid_not_after: domain.x509&.not_after&.iso8601
)
rescue StandardError => e
Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}"
end
end
end
end
end
lib/gitlab/background_migration/fix_pages_access_level.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# corrects stored pages access level on db depending on project visibility
class FixPagesAccessLevel
# Copy routable here to avoid relying on application logic
module Routable
def build_full_path
if parent && path
parent.build_full_path + '/' + path
else
path
end
end
end
# Namespace
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled
include Routable
belongs_to :parent, class_name: "Namespace"
end
# Project
class Project < ActiveRecord::Base
self.table_name = 'projects'
self.inheritance_column = :_type_disabled
include Routable
belongs_to :namespace
alias_method :parent, :namespace
alias_attribute :parent_id, :namespace_id
PRIVATE = 0
INTERNAL = 10
PUBLIC = 20
def pages_deployed?
Dir.exist?(public_pages_path)
end
def public_pages_path
File.join(pages_path, 'public')
end
def pages_path
# TODO: when we migrate Pages to work with new storage types, change here to use disk_path
File.join(Settings.pages.path, build_full_path)
end
end
# ProjectFeature
class ProjectFeature < ActiveRecord::Base
include ::EachBatch
self.table_name = 'project_features'
belongs_to :project
PRIVATE = 10
ENABLED = 20
PUBLIC = 30
end
def perform(start_id, stop_id)
fix_public_access_level(start_id, stop_id)
make_internal_projects_public(start_id, stop_id)
fix_private_access_level(start_id, stop_id)
end
private
def access_control_is_enabled
@access_control_is_enabled = Gitlab.config.pages.access_control
end
# Public projects are allowed to have only enabled pages_access_level
# which is equivalent to public
def fix_public_access_level(start_id, stop_id)
project_features(start_id, stop_id, ProjectFeature::PUBLIC, Project::PUBLIC).each_batch do |features|
features.update_all(pages_access_level: ProjectFeature::ENABLED)
end
end
# If access control is disabled and project has pages deployed
# project will become unavailable when access control will become enabled
# we make these projects public to avoid negative surprise to user
def make_internal_projects_public(start_id, stop_id)
return if access_control_is_enabled
project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::INTERNAL).find_each do |project_feature|
next unless project_feature.project.pages_deployed?
project_feature.update(pages_access_level: ProjectFeature::PUBLIC)
end
end
# Private projects are not allowed to have enabled access level, only `private` and `public`
# If access control is enabled, these projects currently behave as if they have `private` pages_access_level
# if access control is disabled, these projects currently behave as if they have `public` pages_access_level
# so we preserve this behaviour for projects with pages already deployed
# for project without pages we always set `private` access_level
def fix_private_access_level(start_id, stop_id)
project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::PRIVATE).find_each do |project_feature|
if access_control_is_enabled
project_feature.update!(pages_access_level: ProjectFeature::PRIVATE)
else
fixed_access_level = project_feature.project.pages_deployed? ? ProjectFeature::PUBLIC : ProjectFeature::PRIVATE
project_feature.update!(pages_access_level: fixed_access_level)
end
end
end
def project_features(start_id, stop_id, pages_access_level, project_visibility_level)
ProjectFeature.where(id: start_id..stop_id).joins(:project)
.where(pages_access_level: pages_access_level)
.where(projects: { visibility_level: project_visibility_level })
end
end
end
end
lib/gitlab/background_migration/generate_gitlab_subscriptions.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class GenerateGitlabSubscriptions
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_mod_with('Gitlab::BackgroundMigration::GenerateGitlabSubscriptions')
lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# EE-specific migration
class MigrateDevopsSegmentsToGroups
def perform
# no-op for CE
end
end
end
end
Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_mod_with('Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups')
lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This class populates the `finding_uuid` attribute for
# the existing `vulnerability_feedback` records.
class PopulateFindingUuidForVulnerabilityFeedback
REPORT_TYPES = {
sast: 0,
dependency_scanning: 1,
container_scanning: 2,
dast: 3,
secret_detection: 4,
coverage_fuzzing: 5,
api_fuzzing: 6
}.freeze
class VulnerabilityFeedback < ActiveRecord::Base # rubocop:disable Style/Documentation
include EachBatch
self.table_name = 'vulnerability_feedback'
enum category: REPORT_TYPES
scope :in_range, -> (start, stop) { where(id: start..stop) }
scope :without_uuid, -> { where(finding_uuid: nil) }
def self.load_vulnerability_findings
all.to_a.tap { |collection| collection.each(&:vulnerability_finding) }
end
def set_finding_uuid
return unless vulnerability_finding.present? && vulnerability_finding.primary_identifier.present?
update_column(:finding_uuid, calculated_uuid)
rescue StandardError => error
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
end
def vulnerability_finding
BatchLoader.for(finding_key).batch do |finding_keys, loader|
project_ids = finding_keys.map { |key| key[:project_id] }
categories = finding_keys.map { |key| key[:category] }
fingerprints = finding_keys.map { |key| key[:project_fingerprint] }
findings = Finding.with_primary_identifier.where(
project_id: project_ids.uniq,
report_type: categories.uniq,
project_fingerprint: fingerprints.uniq
).to_a
finding_keys.each do |finding_key|
loader.call(
finding_key,
findings.find { |f| finding_key == f.finding_key }
)
end
end
end
private
def calculated_uuid
::Security::VulnerabilityUUID.generate(
report_type: category,
primary_identifier_fingerprint: vulnerability_finding.primary_identifier.fingerprint,
location_fingerprint: vulnerability_finding.location_fingerprint,
project_id: project_id
)
end
def finding_key
{
project_id: project_id,
category: category,
project_fingerprint: project_fingerprint
}
end
end
class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation
include ShaAttribute
self.table_name = 'vulnerability_occurrences'
sha_attribute :project_fingerprint
sha_attribute :location_fingerprint
belongs_to :primary_identifier, class_name: 'Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::Identifier'
enum report_type: REPORT_TYPES
scope :with_primary_identifier, -> { includes(:primary_identifier) }
def finding_key
{
project_id: project_id,
category: report_type,
project_fingerprint: project_fingerprint
}
end
end
class Identifier < ActiveRecord::Base # rubocop:disable Style/Documentation
self.table_name = 'vulnerability_identifiers'
end
def perform(*range)
feedback = VulnerabilityFeedback.without_uuid.in_range(*range).load_vulnerability_findings
feedback.each(&:set_finding_uuid)
log_info(feedback.count)
end
def log_info(feedback_count)
::Gitlab::BackgroundMigration::Logger.info(
migrator: self.class.name,
message: '`finding_uuid` attributes has been set',
count: feedback_count
)
end
end
end
end
lib/gitlab/background_migration/populate_issue_email_participants.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class to migrate service_desk_reply_to email addresses to issue_email_participants
class PopulateIssueEmailParticipants
# rubocop:disable Style/Documentation
class TmpIssue < ActiveRecord::Base
self.table_name = 'issues'
end
def perform(start_id, stop_id)
issues = TmpIssue.select(:id, :service_desk_reply_to, :created_at).where(id: (start_id..stop_id)).where.not(service_desk_reply_to: nil)
rows = issues.map do |issue|
{
issue_id: issue.id,
email: issue.service_desk_reply_to,
created_at: issue.created_at,
updated_at: issue.created_at
}
end
ApplicationRecord.legacy_bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
end
end
end
end
lib/gitlab/background_migration/recalculate_project_authorizations.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop:disable Style/Documentation
class RecalculateProjectAuthorizations
def perform(user_ids)
# no-op
end
end
end
end
lib/gitlab/background_migration/sync_blocking_issues_count.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class SyncBlockingIssuesCount
def perform(start_id, end_id)
end
end
end
end
Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_mod_with('Gitlab::BackgroundMigration::SyncBlockingIssuesCount')
lib/gitlab/background_migration/sync_issues_state_id.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class SyncIssuesStateId
def perform(start_id, end_id)
ActiveRecord::Base.connection.execute <<~SQL
UPDATE issues
SET state_id =
CASE state
WHEN 'opened' THEN 1
WHEN 'closed' THEN 2
END
WHERE state_id IS NULL
AND id BETWEEN #{start_id} AND #{end_id}
SQL
end
end
end
end
lib/gitlab/background_migration/sync_merge_requests_state_id.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class SyncMergeRequestsStateId
def perform(start_id, end_id)
ActiveRecord::Base.connection.execute <<~SQL
UPDATE merge_requests
SET state_id =
CASE state
WHEN 'opened' THEN 1
WHEN 'closed' THEN 2
WHEN 'merged' THEN 3
WHEN 'locked' THEN 4
END
WHERE state_id IS NULL
AND id BETWEEN #{start_id} AND #{end_id}
SQL
end
end
end
end
lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class WrongfullyConfirmedEmailUnconfirmer
class UserModel < ActiveRecord::Base
alias_method :reset, :reload
self.table_name = 'users'
scope :active, -> { where(state: 'active', user_type: nil) } # only humans, skip bots
devise :confirmable
end
class EmailModel < ActiveRecord::Base
alias_method :reset, :reload
self.table_name = 'emails'
belongs_to :user
devise :confirmable
def self.wrongfully_confirmed_emails(start_id, stop_id)
joins(:user)
.merge(UserModel.active)
.where(id: (start_id..stop_id))
.where.not('emails.confirmed_at' => nil)
.where('emails.confirmed_at = users.confirmed_at')
.where('emails.email <> users.email')
.where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)')
end
end
def perform(start_id, stop_id)
email_records = EmailModel
.wrongfully_confirmed_emails(start_id, stop_id)
.to_a
user_ids = email_records.map(&:user_id).uniq
ActiveRecord::Base.transaction do
update_email_records(start_id, stop_id)
update_user_records(user_ids)
end
# Refind the records with the "real" Email model so devise will notice that the user / email is unconfirmed
unconfirmed_email_records = ::Email.where(id: email_records.map(&:id))
ActiveRecord::Associations::Preloader.new.preload(unconfirmed_email_records, [:user])
send_emails(unconfirmed_email_records)
end
private
def update_email_records(start_id, stop_id)
EmailModel.connection.execute <<-SQL
WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{email_query_for_update(start_id, stop_id).to_sql}
)
UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)}
SET confirmed_at = NULL,
confirmation_token = md5_strings.md5_string,
confirmation_sent_at = NOW()
FROM md5_strings
WHERE id = md5_strings.email_id
SQL
end
def update_user_records(user_ids)
UserModel
.where(id: user_ids)
.update_all("confirmed_at = NULL, confirmation_sent_at = NOW(), unconfirmed_email = NULL, confirmation_token=md5(users.id::varchar || users.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}')")
end
def email_query_for_update(start_id, stop_id)
EmailModel
.wrongfully_confirmed_emails(start_id, stop_id)
.select('emails.id as email_id', "md5(emails.id::varchar || emails.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}') as md5_string")
end
def send_emails(email_records)
user_records = email_records.map(&:user).uniq
user_records.each do |user|
Gitlab::BackgroundMigration::Mailers::UnconfirmMailer.unconfirm_notification_email(user).deliver_later
DeviseMailer.confirmation_instructions(user, user.confirmation_token).deliver_later(wait: 1.minute)
end
email_records.each do |email|
DeviseMailer.confirmation_instructions(email, email.confirmation_token).deliver_later(wait: 1.minute)
end
end
end
end
end
spec/lib/gitlab/background_migration/backfill_legacy_project_repositories_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillLegacyProjectRepositories do
it_behaves_like 'backfill migration for project repositories', :legacy
end
spec/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectUpdatedAtAfterRepositoryStorageMove, :migration, schema: 20210301200959 do
let(:projects) { table(:projects) }
let(:project_repository_storage_moves) { table(:project_repository_storage_moves) }
let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
subject { described_class.new }
describe '#perform' do
it 'updates project updated_at column if they were moved to a different repository storage' do
freeze_time do
project_1 = projects.create!(id: 1, namespace_id: namespace.id, updated_at: 1.day.ago)
project_2 = projects.create!(id: 2, namespace_id: namespace.id, updated_at: Time.current)
original_project_3_updated_at = 2.minutes.from_now
project_3 = projects.create!(id: 3, namespace_id: namespace.id, updated_at: original_project_3_updated_at)
original_project_4_updated_at = 10.days.ago
project_4 = projects.create!(id: 4, namespace_id: namespace.id, updated_at: original_project_4_updated_at)
repository_storage_move_1 = project_repository_storage_moves.create!(project_id: project_1.id, updated_at: 2.hours.ago, source_storage_name: 'default', destination_storage_name: 'default')
repository_storage_move_2 = project_repository_storage_moves.create!(project_id: project_2.id, updated_at: Time.current, source_storage_name: 'default', destination_storage_name: 'default')
project_repository_storage_moves.create!(project_id: project_3.id, updated_at: Time.current, source_storage_name: 'default', destination_storage_name: 'default')
subject.perform([1, 2, 3, 4, non_existing_record_id])
expect(project_1.reload.updated_at).to eq(repository_storage_move_1.updated_at + 1.second)
expect(project_2.reload.updated_at).to eq(repository_storage_move_2.updated_at + 1.second)
expect(project_3.reload.updated_at).to eq(original_project_3_updated_at)
expect(project_4.reload.updated_at).to eq(original_project_4_updated_at)
end
end
end
end
spec/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback, schema: 20210301200959 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let(:scanners) { table(:vulnerability_scanners) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:findings) { table(:vulnerability_occurrences) }
let(:vulnerability_feedback) { table(:vulnerability_feedback) }
let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
let(:project) { projects.create!(namespace_id: namespace.id, name: 'foo') }
let(:user) { users.create!(username: 'john.doe', projects_limit: 5) }
let(:scanner) { scanners.create!(project_id: project.id, external_id: 'foo', name: 'bar') }
let(:identifier) { identifiers.create!(project_id: project.id, fingerprint: 'foo', external_type: 'bar', external_id: 'zoo', name: 'baz') }
let(:sast_report) { 0 }
let(:dependency_scanning_report) { 1 }
let(:dast_report) { 3 }
let(:secret_detection_report) { 4 }
let(:project_fingerprint) { Digest::SHA1.hexdigest(SecureRandom.uuid) }
let(:location_fingerprint_1) { Digest::SHA1.hexdigest(SecureRandom.uuid) }
let(:location_fingerprint_2) { Digest::SHA1.hexdigest(SecureRandom.uuid) }
let(:location_fingerprint_3) { Digest::SHA1.hexdigest(SecureRandom.uuid) }
let(:finding_1) { finding_creator.call(sast_report, location_fingerprint_1) }
let(:finding_2) { finding_creator.call(dast_report, location_fingerprint_2) }
let(:finding_3) { finding_creator.call(secret_detection_report, location_fingerprint_3) }
let(:expected_uuid_1) do
Security::VulnerabilityUUID.generate(
report_type: 'sast',
primary_identifier_fingerprint: identifier.fingerprint,
location_fingerprint: location_fingerprint_1,
project_id: project.id
)
end
let(:expected_uuid_2) do
Security::VulnerabilityUUID.generate(
report_type: 'dast',
primary_identifier_fingerprint: identifier.fingerprint,
location_fingerprint: location_fingerprint_2,
project_id: project.id
)
end
let(:expected_uuid_3) do
Security::VulnerabilityUUID.generate(
report_type: 'secret_detection',
primary_identifier_fingerprint: identifier.fingerprint,
location_fingerprint: location_fingerprint_3,
project_id: project.id
)
end
let(:finding_creator) do
-> (report_type, location_fingerprint) do
findings.create!(
project_id: project.id,
primary_identifier_id: identifier.id,
scanner_id: scanner.id,
report_type: report_type,
uuid: SecureRandom.uuid,
name: 'Foo',
location_fingerprint: Gitlab::Database::ShaAttribute.serialize(location_fingerprint),
project_fingerprint: Gitlab::Database::ShaAttribute.serialize(project_fingerprint),
metadata_version: '1',
severity: 0,
confidence: 5,
raw_metadata: '{}'
)
end
end
let(:feedback_creator) do
-> (category, project_fingerprint) do
vulnerability_feedback.create!(
project_id: project.id,
author_id: user.id,
feedback_type: 0,
category: category,
project_fingerprint: project_fingerprint
)
end
end
let!(:feedback_1) { feedback_creator.call(finding_1.report_type, project_fingerprint) }
let!(:feedback_2) { feedback_creator.call(finding_2.report_type, project_fingerprint) }
let!(:feedback_3) { feedback_creator.call(finding_3.report_type, project_fingerprint) }
let!(:feedback_4) { feedback_creator.call(finding_1.report_type, 'foo') }
let!(:feedback_5) { feedback_creator.call(dependency_scanning_report, project_fingerprint) }
subject(:populate_finding_uuids) { described_class.new.perform(feedback_1.id, feedback_5.id) }
before do
allow(Gitlab::BackgroundMigration::Logger).to receive(:info)
end
describe '#perform' do
it 'updates the `finding_uuid` attributes of the feedback records' do
expect { populate_finding_uuids }.to change { feedback_1.reload.finding_uuid }.from(nil).to(expected_uuid_1)
.and change { feedback_2.reload.finding_uuid }.from(nil).to(expected_uuid_2)
.and change { feedback_3.reload.finding_uuid }.from(nil).to(expected_uuid_3)
.and not_change { feedback_4.reload.finding_uuid }
.and not_change { feedback_5.reload.finding_uuid }
expect(Gitlab::BackgroundMigration::Logger).to have_received(:info).once
end
it 'preloads the finding and identifier records to prevent N+1 queries' do
# Load feedback records(1), load findings(2), load identifiers(3) and finally update feedback records one by one(6)
expect { populate_finding_uuids }.not_to exceed_query_limit(6)
end
context 'when setting the `finding_uuid` attribute of a feedback record fails' do
let(:expected_error) { RuntimeError.new }
before do
allow(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception)
allow_next_found_instance_of(described_class::VulnerabilityFeedback) do |feedback|
allow(feedback).to receive(:update_column).and_raise(expected_error)
end
end
it 'captures the errors and does not crash entirely' do
expect { populate_finding_uuids }.not_to raise_error
expect(Gitlab::ErrorTracking).to have_received(:track_and_raise_for_dev_exception).with(expected_error).exactly(3).times
end
end
end
end
spec/lib/gitlab/background_migration/populate_issue_email_participants_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateIssueEmailParticipants, schema: 20210301200959 do
let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') }
let!(:project) { table(:projects).create!(id: 1, namespace_id: namespace.id) }
let!(:issue1) { table(:issues).create!(id: 1, project_id: project.id, service_desk_reply_to: "a@gitlab.com") }
let!(:issue2) { table(:issues).create!(id: 2, project_id: project.id, service_desk_reply_to: "b@gitlab.com") }
let(:issue_email_participants) { table(:issue_email_participants) }
describe '#perform' do
it 'migrates email addresses from service desk issues', :aggregate_failures do
expect { subject.perform(1, 2) }.to change { issue_email_participants.count }.by(2)
expect(issue_email_participants.find_by(issue_id: 1).email).to eq("a@gitlab.com")
expect(issue_email_participants.find_by(issue_id: 2).email).to eq("b@gitlab.com")
end
end
end
spec/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer_spec.rb deleted 100644 → 0
View file @ 9e54019b
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::WrongfullyConfirmedEmailUnconfirmer, schema: 20210301200959 do
let(:users) { table(:users) }
let(:emails) { table(:emails) }
let(:user_synced_attributes_metadata) { table(:user_synced_attributes_metadata) }
let(:confirmed_at_2_days_ago) { 2.days.ago }
let(:confirmed_at_3_days_ago) { 3.days.ago }
let(:one_year_ago) { 1.year.ago }
let!(:user_needs_migration_1) { users.create!(name: 'user1', email: 'test1@test.com', state: 'active', projects_limit: 1, confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
let!(:user_needs_migration_2) { users.create!(name: 'user2', email: 'test2@test.com', unconfirmed_email: 'unconfirmed@test.com', state: 'active', projects_limit: 1, confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:user_does_not_need_migration) { users.create!(name: 'user3', email: 'test3@test.com', state: 'active', projects_limit: 1) }
let!(:inactive_user) { users.create!(name: 'user4', email: 'test4@test.com', state: 'blocked', projects_limit: 1, confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:alert_bot_user) { users.create!(name: 'user5', email: 'test5@test.com', state: 'active', user_type: 2, projects_limit: 1, confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:user_has_synced_email) { users.create!(name: 'user6', email: 'test6@test.com', state: 'active', projects_limit: 1, confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
let!(:synced_attributes_metadata_for_user) { user_synced_attributes_metadata.create!(user_id: user_has_synced_email.id, email_synced: true) }
let!(:bad_email_1) { emails.create!(user_id: user_needs_migration_1.id, email: 'other1@test.com', confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
let!(:bad_email_2) { emails.create!(user_id: user_needs_migration_2.id, email: 'other2@test.com', confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:bad_email_3_inactive_user) { emails.create!(user_id: inactive_user.id, email: 'other-inactive@test.com', confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:bad_email_4_bot_user) { emails.create!(user_id: alert_bot_user.id, email: 'other-bot@test.com', confirmed_at: confirmed_at_3_days_ago, confirmation_sent_at: one_year_ago) }
let!(:good_email_1) { emails.create!(user_id: user_needs_migration_2.id, email: 'other3@test.com', confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
let!(:good_email_2) { emails.create!(user_id: user_needs_migration_2.id, email: 'other4@test.com', confirmed_at: nil) }
let!(:good_email_3) { emails.create!(user_id: user_does_not_need_migration.id, email: 'other5@test.com', confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
let!(:second_email_for_user_with_synced_email) { emails.create!(user_id: user_has_synced_email.id, email: 'other6@test.com', confirmed_at: confirmed_at_2_days_ago, confirmation_sent_at: one_year_ago) }
subject do
email_ids = [bad_email_1, bad_email_2, good_email_1, good_email_2, good_email_3, second_email_for_user_with_synced_email].map(&:id)
described_class.new.perform(email_ids.min, email_ids.max)
end
it 'does not change irrelevant email records' do
subject
expect(good_email_1.reload.confirmed_at).to be_within(1.second).of(confirmed_at_2_days_ago)
expect(good_email_2.reload.confirmed_at).to be_nil
expect(good_email_3.reload.confirmed_at).to be_within(1.second).of(confirmed_at_2_days_ago)
expect(bad_email_3_inactive_user.reload.confirmed_at).to be_within(1.second).of(confirmed_at_3_days_ago)
expect(bad_email_4_bot_user.reload.confirmed_at).to be_within(1.second).of(confirmed_at_3_days_ago)
expect(good_email_1.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
expect(good_email_2.reload.confirmation_sent_at).to be_nil
expect(good_email_3.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
expect(bad_email_3_inactive_user.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
expect(bad_email_4_bot_user.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
end
it 'clears the `unconfirmed_email` field' do
subject
user_needs_migration_2.reload
expect(user_needs_migration_2.unconfirmed_email).to be_nil
end
it 'does not change irrelevant user records' do
subject
expect(user_does_not_need_migration.reload.confirmed_at).to be_nil
expect(inactive_user.reload.confirmed_at).to be_within(1.second).of(confirmed_at_3_days_ago)
expect(alert_bot_user.reload.confirmed_at).to be_within(1.second).of(confirmed_at_3_days_ago)
expect(user_has_synced_email.reload.confirmed_at).to be_within(1.second).of(confirmed_at_2_days_ago)
expect(user_does_not_need_migration.reload.confirmation_sent_at).to be_nil
expect(inactive_user.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
expect(alert_bot_user.reload.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
expect(user_has_synced_email.confirmation_sent_at).to be_within(1.second).of(one_year_ago)
end
it 'updates confirmation_sent_at column' do
subject
expect(user_needs_migration_1.reload.confirmation_sent_at).to be_within(1.minute).of(Time.now)
expect(user_needs_migration_2.reload.confirmation_sent_at).to be_within(1.minute).of(Time.now)
expect(bad_email_1.reload.confirmation_sent_at).to be_within(1.minute).of(Time.now)
expect(bad_email_2.reload.confirmation_sent_at).to be_within(1.minute).of(Time.now)
end
it 'unconfirms bad email records' do
subject
expect(bad_email_1.reload.confirmed_at).to be_nil
expect(bad_email_2.reload.confirmed_at).to be_nil
expect(bad_email_1.reload.confirmation_token).not_to be_nil
expect(bad_email_2.reload.confirmation_token).not_to be_nil
end
it 'unconfirms user records' do
subject
expect(user_needs_migration_1.reload.confirmed_at).to be_nil
expect(user_needs_migration_2.reload.confirmed_at).to be_nil
expect(user_needs_migration_1.reload.confirmation_token).not_to be_nil
expect(user_needs_migration_2.reload.confirmation_token).not_to be_nil
end
context 'enqueued jobs' do
let(:user_1) { User.find(user_needs_migration_1.id) }
let(:user_2) { User.find(user_needs_migration_2.id) }
let(:email_1) { Email.find(bad_email_1.id) }
let(:email_2) { Email.find(bad_email_2.id) }
it 'enqueues the email confirmation and the unconfirm notification mailer jobs' do
allow(DeviseMailer).to receive(:confirmation_instructions).and_call_original
allow(Gitlab::BackgroundMigration::Mailers::UnconfirmMailer).to receive(:unconfirm_notification_email).and_call_original
subject
expect(DeviseMailer).to have_received(:confirmation_instructions).with(email_1, email_1.confirmation_token)
expect(DeviseMailer).to have_received(:confirmation_instructions).with(email_2, email_2.confirmation_token)
expect(Gitlab::BackgroundMigration::Mailers::UnconfirmMailer).to have_received(:unconfirm_notification_email).with(user_1)
expect(DeviseMailer).to have_received(:confirmation_instructions).with(user_1, user_1.confirmation_token)
expect(Gitlab::BackgroundMigration::Mailers::UnconfirmMailer).to have_received(:unconfirm_notification_email).with(user_2)
expect(DeviseMailer).to have_received(:confirmation_instructions).with(user_2, user_2.confirmation_token)
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7