Skip to content

G
gitlab-ce
Commit f36ab2c2 authored by Andreas Brandl's avatar Andreas Brandl
Browse files
Options

Delete unused background migration code 

This cleans up and removes background migrations we don't use anymore.
This is related to the cleanup of regular migrations in
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31936.

Closes https://gitlab.com/gitlab-org/gitlab/-/issues/218998
parent 9b491d6a
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1516 deletions
lib/gitlab/background_migration/archive_legacy_traces.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class ArchiveLegacyTraces
def perform(start_id, stop_id)
# This background migration directly refers to ::Ci::Build model which is defined in application code.
# In general, migration code should be isolated as much as possible in order to be idempotent.
# However, `archive!` method is too complicated to be replicated by coping its subsequent code.
# So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1
::Ci::Build.finished.without_archived_trace
.where(id: start_id..stop_id).find_each do |build|
build.trace.archive!
rescue => e
Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger
end
end
end
end
end
lib/gitlab/background_migration/backfill_hashed_project_repositories.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class that will fill the project_repositories table for projects that
# are on hashed storage and an entry is is missing in this table.
class BackfillHashedProjectRepositories < BackfillProjectRepositories
private
def projects
Project.on_hashed_storage
end
end
end
end
lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This module is used to write the full path of all projects to
# the git repository config file.
# Storing the full project path in the git config allows admins to
# easily identify a project when it is using hashed storage.
module BackfillProjectFullpathInRepoConfig
OrphanedNamespaceError = Class.new(StandardError)
module Storage
# Class that returns the disk path for a project using hashed storage
class Hashed
attr_accessor :project
ROOT_PATH_PREFIX = '@hashed'
def initialize(project)
@project = project
end
def disk_path
"#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
end
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
end
end
# Class that returns the disk path for a project using legacy storage
class LegacyProject
attr_accessor :project
def initialize(project)
@project = project
end
def disk_path
project.full_path
end
end
end
# Concern used by Project and Namespace to determine the full
# route to the project
module Routable
extend ActiveSupport::Concern
def full_path
@full_path ||= build_full_path
end
def build_full_path
return path unless has_parent?
raise OrphanedNamespaceError if parent.nil?
parent.full_path + '/' + path
end
def has_parent?
read_attribute(association(:parent).reflection.foreign_key)
end
end
# Class used to interact with repository using Gitaly
class Repository
attr_reader :storage
def initialize(storage, relative_path)
@storage = storage
@relative_path = relative_path
end
def gitaly_repository
Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
end
end
# Namespace can be a user or group. It can be the root or a
# child of another namespace.
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = nil
include Routable
belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
has_many :projects, inverse_of: :parent
has_many :namespaces, inverse_of: :parent
end
# Project is where the repository (etc.) is stored
class Project < ActiveRecord::Base
self.table_name = 'projects'
include Routable
include EachBatch
FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
delegate :disk_path, to: :storage
def add_fullpath_config
entries = { FULLPATH_CONFIG_KEY => full_path }
repository_service.set_config(entries)
end
def remove_fullpath_config
repository_service.delete_config([FULLPATH_CONFIG_KEY])
end
def cleanup_repository
repository_service.cleanup
end
def storage
@storage ||=
if hashed_storage?
Storage::Hashed.new(self)
else
Storage::LegacyProject.new(self)
end
end
def hashed_storage?
self.storage_version && self.storage_version >= 1
end
def repository
@repository ||= Repository.new(repository_storage, disk_path + '.git')
end
def repository_service
@repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
end
end
# Base class for Up and Down migration classes
class BackfillFullpathMigration
RETRY_DELAY = 15.minutes
MAX_RETRIES = 2
# Base class for retrying one project
class BaseRetryOne
def perform(project_id, retry_count)
project = Project.find(project_id)
return unless project
migration_class.new.safe_perform_one(project, retry_count)
end
end
def perform(start_id, end_id)
Project.includes(:parent).where(id: start_id..end_id).each do |project|
safe_perform_one(project)
end
end
def safe_perform_one(project, retry_count = 0)
perform_one(project)
rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError
nil
rescue GRPC::BadStatus
schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES
end
def schedule_retry(project, retry_count)
# Constants provided to BackgroundMigrationWorker must be within the
# scope of Gitlab::BackgroundMigration
retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '')
BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count])
end
end
# Class to add the fullpath to the git repo config
class Up < BackfillFullpathMigration
# Class used to retry
class RetryOne < BaseRetryOne
def migration_class
Up
end
end
def perform_one(project)
project.cleanup_repository
project.add_fullpath_config
end
end
# Class to rollback adding the fullpath to the git repo config
class Down < BackfillFullpathMigration
# Class used to retry
class RetryOne < BaseRetryOne
def migration_class
Down
end
end
def perform_one(project)
project.cleanup_repository
project.remove_fullpath_config
end
end
end
end
end
lib/gitlab/background_migration/fill_file_store_job_artifact.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillFileStoreJobArtifact
class JobArtifact < ActiveRecord::Base
self.table_name = 'ci_job_artifacts'
end
def perform(start_id, stop_id)
FillFileStoreJobArtifact::JobArtifact
.where(file_store: nil)
.where(id: (start_id..stop_id))
.update_all(file_store: 1)
end
end
end
end
lib/gitlab/background_migration/fill_file_store_lfs_object.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillFileStoreLfsObject
class LfsObject < ActiveRecord::Base
self.table_name = 'lfs_objects'
end
def perform(start_id, stop_id)
FillFileStoreLfsObject::LfsObject
.where(file_store: nil)
.where(id: (start_id..stop_id))
.update_all(file_store: 1)
end
end
end
end
lib/gitlab/background_migration/fill_store_upload.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillStoreUpload
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
self.inheritance_column = :_type_disabled
end
def perform(start_id, stop_id)
FillStoreUpload::Upload
.where(store: nil)
.where(id: (start_id..stop_id))
.update_all(store: 1)
end
end
end
end
lib/gitlab/background_migration/fix_cross_project_label_links.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FixCrossProjectLabelLinks
GROUP_NESTED_LEVEL = 10.freeze
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
class Label < ActiveRecord::Base
self.inheritance_column = :_type_disabled
self.table_name = 'labels'
end
class LabelLink < ActiveRecord::Base
self.table_name = 'label_links'
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
class Namespace < ActiveRecord::Base
self.inheritance_column = :_type_disabled
self.table_name = 'namespaces'
def self.groups_with_descendants_ids(start_id, stop_id)
# To isolate migration code, we avoid usage of
# Gitlab::GroupHierarchy#base_and_descendants which already
# does this job better
ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id)
group_ids = ids
GROUP_NESTED_LEVEL.times do
ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id)
break if ids.empty?
group_ids += ids
end
group_ids.uniq
end
end
def perform(start_id, stop_id)
group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id)
project_ids = Project.where(namespace_id: group_ids).select(:id)
fix_issues(project_ids)
fix_merge_requests(project_ids)
end
private
# select IDs of issues which reference a label which is:
# a) a project label of a different project, or
# b) a group label of a different group than issue's project group
def fix_issues(project_ids)
issue_ids = Label
.joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\'
INNER JOIN issues ON issues.id = label_links.target_id
INNER JOIN projects ON projects.id = issues.project_id')
.where('issues.project_id in (?)', project_ids)
.where('(labels.project_id is not null and labels.project_id != issues.project_id) '\
'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
.select('distinct issues.id')
Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) }
end
# select IDs of MRs which reference a label which is:
# a) a project label of a different project, or
# b) a group label of a different group than MR's project group
def fix_merge_requests(project_ids)
mr_ids = Label
.joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\'
INNER JOIN merge_requests ON merge_requests.id = label_links.target_id
INNER JOIN projects ON projects.id = merge_requests.target_project_id')
.where('merge_requests.target_project_id in (?)', project_ids)
.where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\
'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
.select('distinct merge_requests.id')
MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) }
end
def check_resource_labels(resource, project_id)
local_labels = available_labels(project_id)
# get all label links for the given resource (issue/MR)
# which reference a label not included in available_labels
# (other than its project labels and labels of ancestor groups)
cross_labels = LabelLink
.select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id')
.joins('INNER JOIN labels ON labels.id = label_links.label_id')
.where(target_type: resource.class.name.demodulize, target_id: resource.id)
.where('labels.id not in (?)', local_labels.select(:id))
cross_labels.each do |label|
matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color}
next unless matching_label
Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger
LabelLink.update(label.label_link_id, label_id: matching_label.id)
end
end
# get all labels available for the project (including
# group labels of ancestor groups)
def available_labels(project_id)
@labels ||= {}
@labels[project_id] ||= Label
.where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)",
project_group_ids(project_id),
project_id)
end
def project_group_ids(project_id)
ids = [Project.find(project_id).namespace_id]
GROUP_NESTED_LEVEL.times do
group = Namespace.find(ids.last)
break unless group.parent_id
ids << group.parent_id
end
ids
end
end
end
end
lib/gitlab/background_migration/migrate_build_stage.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateBuildStage
module Migratable
class Stage < ActiveRecord::Base
self.table_name = 'ci_stages'
end
class Build < ActiveRecord::Base
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
def ensure_stage!(attempts: 2)
find_stage || create_stage!
rescue ActiveRecord::RecordNotUnique
retry if (attempts -= 1) > 0
raise
end
def find_stage
Stage.find_by(name: self.stage || 'test',
pipeline_id: self.commit_id,
project_id: self.project_id)
end
def create_stage!
Stage.create!(name: self.stage || 'test',
pipeline_id: self.commit_id,
project_id: self.project_id)
end
end
end
def perform(start_id, stop_id)
stages = Migratable::Build.where('stage_id IS NULL')
.where('id BETWEEN ? AND ?', start_id, stop_id)
.map { |build| build.ensure_stage! }
.compact.map(&:id)
MigrateBuildStageIdReference.new.perform(start_id, stop_id)
MigrateStageStatus.new.perform(stages.min, stages.max)
end
end
end
end
lib/gitlab/background_migration/migrate_build_stage_id_reference.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateBuildStageIdReference
def perform(start_id, stop_id)
sql = <<-SQL.strip_heredoc
UPDATE ci_builds
SET stage_id =
(SELECT id FROM ci_stages
WHERE ci_stages.pipeline_id = ci_builds.commit_id
AND ci_stages.name = ci_builds.stage)
WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
AND ci_builds.stage_id IS NULL
SQL
ActiveRecord::Base.connection.execute(sql)
end
end
end
end
lib/gitlab/background_migration/migrate_stage_index.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateStageIndex
def perform(start_id, stop_id)
migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql|
ActiveRecord::Base.connection.execute(sql)
end
end
private
def migrate_stage_index_sql(start_id, stop_id)
<<~SQL
WITH freqs AS (
SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds
WHERE stage_id BETWEEN #{start_id} AND #{stop_id}
AND stage_idx IS NOT NULL
GROUP BY stage_id, stage_idx
), indexes AS (
SELECT DISTINCT stage_id, first_value(stage_idx)
OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index
FROM freqs
)
UPDATE ci_stages SET position = indexes.index
FROM indexes WHERE indexes.stage_id = ci_stages.id
AND ci_stages.position IS NULL;
SQL
end
end
end
end
lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
#
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class PopulateClusterKubernetesNamespaceTable
include Gitlab::Database::MigrationHelpers
BATCH_SIZE = 1_000
module Migratable
class KubernetesNamespace < ActiveRecord::Base
self.table_name = 'clusters_kubernetes_namespaces'
end
class ClusterProject < ActiveRecord::Base
include EachBatch
self.table_name = 'cluster_projects'
belongs_to :project
def self.with_no_kubernetes_namespace
where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id))
end
def namespace
slug = "#{project.path}-#{project.id}".downcase
slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
end
def service_account
"#{namespace}-service-account"
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
end
def perform
cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index|
sql_values = sql_values_for(cluster_projects_batch)
insert_into_cluster_kubernetes_namespace(sql_values)
end
end
private
def cluster_projects_with_no_kubernetes_namespace
Migratable::ClusterProject.with_no_kubernetes_namespace
end
def sql_values_for(cluster_projects)
cluster_projects.map do |cluster_project|
values_for_cluster_project(cluster_project)
end
end
def values_for_cluster_project(cluster_project)
{
cluster_project_id: cluster_project.id,
cluster_id: cluster_project.cluster_id,
project_id: cluster_project.project_id,
namespace: cluster_project.namespace,
service_account_name: cluster_project.service_account,
created_at: 'NOW()',
updated_at: 'NOW()'
}
end
def insert_into_cluster_kubernetes_namespace(rows)
Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert
rows,
disable_quote: [:created_at, :updated_at])
end
end
end
end
lib/gitlab/background_migration/remove_restricted_todos.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
# rubocop:disable Metrics/ClassLength
module Gitlab
module BackgroundMigration
class RemoveRestrictedTodos
PRIVATE_FEATURE = 10
PRIVATE_PROJECT = 0
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
class ProjectAuthorization < ActiveRecord::Base
self.table_name = 'project_authorizations'
end
class ProjectFeature < ActiveRecord::Base
self.table_name = 'project_features'
end
class Todo < ActiveRecord::Base
include EachBatch
self.table_name = 'todos'
end
class Issue < ActiveRecord::Base
include EachBatch
self.table_name = 'issues'
end
def perform(start_id, stop_id)
projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)')
.where(id: start_id..stop_id)
projects.each do |project|
remove_confidential_issue_todos(project.id)
if project.visibility_level == PRIVATE_PROJECT
remove_non_members_todos(project.id)
else
remove_restricted_features_todos(project.id)
end
end
end
private
def remove_non_members_todos(project_id)
batch_remove_todos_cte(project_id)
end
def remove_confidential_issue_todos(project_id)
# min access level to access a confidential issue is reporter
min_reporters = authorized_users(project_id)
.select(:user_id)
.where('access_level >= ?', 20)
confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id)
confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch|
batch.each do |issue|
assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id)
todos = Todo.where(target_type: 'Issue', target_id: issue.id)
.where('user_id NOT IN (?)', min_reporters)
.where('user_id NOT IN (?)', assigned_users)
todos = todos.where('user_id != ?', issue.author_id) if issue.author_id
todos.delete_all
end
end
end
def remove_restricted_features_todos(project_id)
ProjectFeature.where(project_id: project_id).each do |project_features|
target_types = []
target_types << 'Issue' if private?(project_features.issues_access_level)
target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level)
target_types << 'Commit' if private?(project_features.repository_access_level)
next if target_types.empty?
batch_remove_todos_cte(project_id, target_types)
end
end
def private?(feature_level)
feature_level == PRIVATE_FEATURE
end
def authorized_users(project_id)
ProjectAuthorization.select(:user_id).where(project_id: project_id)
end
def unauthorized_project_todos(project_id)
Todo.where(project_id: project_id)
.where('user_id NOT IN (?)', authorized_users(project_id))
end
def batch_remove_todos_cte(project_id, target_types = nil)
loop do
count = remove_todos_cte(project_id, target_types)
break if count == 0
end
end
def remove_todos_cte(project_id, target_types = nil)
sql = []
sql << with_all_todos_sql(project_id, target_types)
sql << as_deleted_sql
sql << "SELECT count(*) FROM deleted"
result = Todo.connection.exec_query(sql.join(' '))
result.rows[0][0].to_i
end
def with_all_todos_sql(project_id, target_types = nil)
if target_types
table = Arel::Table.new(:todos)
in_target = table[:target_type].in(target_types)
target_types_sql = " AND #{in_target.to_sql}"
end
<<-SQL
WITH all_todos AS (
SELECT id
FROM "todos"
WHERE "todos"."project_id" = #{project_id}
AND (user_id NOT IN (
SELECT "project_authorizations"."user_id"
FROM "project_authorizations"
WHERE "project_authorizations"."project_id" = #{project_id})
#{target_types_sql}
)
),
SQL
end
def as_deleted_sql
<<-SQL
deleted AS (
DELETE FROM todos
WHERE id IN (
SELECT id
FROM all_todos
LIMIT 5000
)
RETURNING id
)
SQL
end
end
end
end
lib/gitlab/background_migration/set_confidential_note_events_on_services.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
# Ensures services which previously received all notes events continue
# to receive confidential ones.
class SetConfidentialNoteEventsOnServices
class Service < ActiveRecord::Base
self.table_name = 'services'
include ::EachBatch
def self.services_to_update
where(confidential_note_events: nil, note_events: true)
end
end
def perform(start_id, stop_id)
Service.services_to_update
.where(id: start_id..stop_id)
.update_all(confidential_note_events: true)
end
end
end
end
lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
# Ensures hooks which previously received all notes events continue
# to receive confidential ones.
class SetConfidentialNoteEventsOnWebhooks
class WebHook < ActiveRecord::Base
self.table_name = 'web_hooks'
include ::EachBatch
def self.hooks_to_update
where(confidential_note_events: nil, note_events: true)
end
end
def perform(start_id, stop_id)
WebHook.hooks_to_update
.where(id: start_id..stop_id)
.update_all(confidential_note_events: true)
end
end
end
end
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces do
include TraceHelpers
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:builds) { table(:ci_builds) }
let(:job_artifacts) { table(:ci_job_artifacts) }
before do
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
@build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build')
end
context 'when trace file exsits at the right place' do
before do
create_legacy_trace(@build, 'trace in file')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(File.exist?(legacy_trace_path(@build))).to be_truthy
described_class.new.perform(1, 1)
expect(job_artifacts.count).to eq(1)
expect(File.exist?(legacy_trace_path(@build))).to be_falsy
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file')
end
end
context 'when trace file does not exsits at the right place' do
it 'does not raise errors nor create job artifact' do
expect { described_class.new.perform(1, 1) }.not_to raise_error
expect(job_artifacts.count).to eq(0)
end
end
context 'when trace data exsits in database' do
before do
create_legacy_trace_in_db(@build, 'trace in db')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(@build.read_attribute(:trace)).not_to be_empty
described_class.new.perform(1, 1)
@build.reload
expect(job_artifacts.count).to eq(1)
expect(@build.read_attribute(:trace)).to be_nil
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db')
end
end
end
spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories do
it_behaves_like 'backfill migration for project repositories', :hashed
end
spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
let(:subgroup) { namespaces.create!(name: 'bar', path: 'bar', parent_id: group.id) }
describe described_class::Storage::Hashed do
let(:project) { double(id: 555) }
subject(:project_storage) { described_class.new(project) }
it 'has the correct disk_path' do
expect(project_storage.disk_path).to eq('@hashed/91/a7/91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8')
end
end
describe described_class::Storage::LegacyProject do
let(:project) { double(full_path: 'this/is/the/full/path') }
subject(:project_storage) { described_class.new(project) }
it 'has the correct disk_path' do
expect(project_storage.disk_path).to eq('this/is/the/full/path')
end
end
describe described_class::Project do
let(:project_record) { projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') }
subject(:project) { described_class.find(project_record.id) }
describe '#full_path' do
it 'returns path containing all parent namespaces' do
expect(project.full_path).to eq('foo/bar/baz')
end
it 'raises OrphanedNamespaceError when any parent namespace does not exist' do
subgroup.update_attribute(:parent_id, non_existing_record_id)
expect { project.full_path }.to raise_error(Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig::OrphanedNamespaceError)
end
end
end
describe described_class::Up do
describe '#perform' do
subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) }
it 'asks the gitaly client to set config' do
projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz')
projects.create!(namespace_id: subgroup.id, name: 'buzz', path: 'buzz', storage_version: 1)
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/baz')
end
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/buzz')
end
migrate
end
end
end
describe described_class::Down do
describe '#perform' do
subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) }
it 'asks the gitaly client to set config' do
projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz')
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:delete_config).with(['gitlab.fullpath'])
end
migrate
end
end
end
end
spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks do
let(:namespaces_table) { table(:namespaces) }
let(:projects_table) { table(:projects) }
let(:issues_table) { table(:issues) }
let(:merge_requests_table) { table(:merge_requests) }
let(:labels_table) { table(:labels) }
let(:label_links_table) { table(:label_links) }
let!(:group1) { namespaces_table.create(id: 10, type: 'Group', name: 'group1', path: 'group1') }
let!(:group2) { namespaces_table.create(id: 20, type: 'Group', name: 'group2', path: 'group2') }
let!(:project1) { projects_table.create(id: 1, name: 'project1', path: 'group1/project1', namespace_id: 10) }
let!(:project2) { projects_table.create(id: 3, name: 'project2', path: 'group1/project2', namespace_id: 20) }
let!(:label1) { labels_table.create(id: 1, title: 'bug', color: 'red', group_id: 10, type: 'GroupLabel') }
let!(:label2) { labels_table.create(id: 2, title: 'bug', color: 'red', group_id: 20, type: 'GroupLabel') }
def create_merge_request(id, project_id)
merge_requests_table.create(id: id,
target_project_id: project_id,
target_branch: 'master',
source_project_id: project_id,
source_branch: 'mr name',
title: "mr name#{id}")
end
def create_issue(id, project_id)
issues_table.create(id: id, title: "issue#{id}", project_id: project_id)
end
def create_resource(target_type, id, project_id)
target_type == 'Issue' ? create_issue(id, project_id) : create_merge_request(id, project_id)
end
shared_examples_for 'resource with cross-project labels' do
it 'updates only cross-project label links which exist in the local project or group' do
create_resource(target_type, 1, 1)
create_resource(target_type, 2, 3)
labels_table.create(id: 3, title: 'bug', color: 'red', project_id: 3, type: 'ProjectLabel')
link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1)
link2 = label_links_table.create(label_id: 3, target_type: target_type, target_id: 2)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
expect(link2.reload.label_id).to eq(3)
end
it 'ignores cross-project label links if label color is different' do
labels_table.create(id: 3, title: 'bug', color: 'green', group_id: 20, type: 'GroupLabel')
create_resource(target_type, 1, 1)
link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(3)
end
it 'ignores cross-project label links if label name is different' do
labels_table.create(id: 3, title: 'bug1', color: 'red', group_id: 20, type: 'GroupLabel')
create_resource(target_type, 1, 1)
link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(3)
end
context 'with nested group' do
before do
namespaces_table.create(id: 11, type: 'Group', name: 'subgroup1', path: 'group1/subgroup1', parent_id: 10)
projects_table.create(id: 2, name: 'subproject1', path: 'group1/subgroup1/subproject1', namespace_id: 11)
create_resource(target_type, 1, 2)
end
it 'ignores label links referencing ancestor group labels' do
labels_table.create(id: 4, title: 'bug', color: 'red', project_id: 2, type: 'ProjectLabel')
label_links_table.create(label_id: 4, target_type: target_type, target_id: 1)
link = label_links_table.create(label_id: 1, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
end
it 'checks also issues and MRs in subgroups' do
link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
end
end
end
context 'resource is Issue' do
it_behaves_like 'resource with cross-project labels' do
let(:target_type) { 'Issue' }
end
end
context 'resource is Merge Request' do
it_behaves_like 'resource with cross-project labels' do
let(:target_type) { 'MergeRequest' }
end
end
end
spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
let(:jobs) { table(:ci_builds) }
let(:statuses) do
{
created: 0,
pending: 1,
running: 2,
success: 3,
failed: 4,
canceled: 5,
skipped: 6,
manual: 7
}
end
before do
namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id)
pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
jobs.create!(id: 1, commit_id: 1, project_id: 123,
stage_idx: 2, stage: 'build', status: :success)
jobs.create!(id: 2, commit_id: 1, project_id: 123,
stage_idx: 2, stage: 'build', status: :success)
jobs.create!(id: 3, commit_id: 1, project_id: 123,
stage_idx: 1, stage: 'test', status: :failed)
jobs.create!(id: 4, commit_id: 1, project_id: 123,
stage_idx: 1, stage: 'test', status: :success)
jobs.create!(id: 5, commit_id: 1, project_id: 123,
stage_idx: 3, stage: 'deploy', status: :pending)
jobs.create!(id: 6, commit_id: 1, project_id: 123,
stage_idx: 3, stage: nil, status: :pending)
end
it 'correctly migrates builds stages' do
expect(stages.count).to be_zero
described_class.new.perform(1, 6)
expect(stages.count).to eq 3
expect(stages.all.pluck(:name)).to match_array %w[test build deploy]
expect(jobs.where(stage_id: nil)).to be_one
expect(jobs.find_by(stage_id: nil).id).to eq 6
expect(stages.all.pluck(:status)).to match_array [statuses[:success],
statuses[:failed],
statuses[:pending]]
end
it 'recovers from unique constraint violation only twice', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/28128' do
allow(described_class::Migratable::Stage)
.to receive(:find_by).and_return(nil)
expect(described_class::Migratable::Stage)
.to receive(:find_by).exactly(3).times
expect { described_class.new.perform(1, 6) }
.to raise_error ActiveRecord::RecordNotUnique
end
context 'when invalid class can be loaded due to single table inheritance' do
let(:commit_status) do
jobs.create!(id: 7, commit_id: 1, project_id: 123, stage_idx: 4,
stage: 'post-deploy', status: :failed)
end
before do
commit_status.update_column(:type, 'SomeClass')
end
it 'does ignore single table inheritance type' do
expect { described_class.new.perform(1, 7) }.not_to raise_error
expect(jobs.find(7)).to have_attributes(stage_id: (a_value > 0))
end
end
end
spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
let(:jobs) { table(:ci_builds) }
let(:namespace) { namespaces.create(name: 'gitlab-org', path: 'gitlab-org') }
let(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') }
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
let(:stage1) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'build') }
let(:stage2) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'test') }
before do
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 10, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 3, stage_id: stage2.id)
end
it 'correctly migrates stages indices' do
expect(stages.all.pluck(:position)).to all(be_nil)
described_class.new.perform(stage1.id, stage2.id)
expect(stages.all.order(:id).pluck(:position)).to eq [2, 3]
end
end
spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable do
include MigrationHelpers::ClusterHelpers
let(:migration) { described_class.new }
let(:clusters_table) { table(:clusters) }
let(:cluster_projects_table) { table(:cluster_projects) }
let(:cluster_kubernetes_namespaces_table) { table(:clusters_kubernetes_namespaces) }
let(:projects_table) { table(:projects) }
let(:namespaces_table) { table(:namespaces) }
let(:provider_gcp_table) { table(:cluster_providers_gcp) }
let(:platform_kubernetes_table) { table(:cluster_platforms_kubernetes) }
before do
create_cluster_project_list(10)
end
shared_examples 'consistent kubernetes namespace attributes' do
it 'populates namespace and service account information' do
migration.perform
clusters_with_namespace.each do |cluster|
cluster_project = cluster_projects_table.find_by(cluster_id: cluster.id)
project = projects_table.find(cluster_project.project_id)
kubernetes_namespace = cluster_kubernetes_namespaces_table.find_by(cluster_id: cluster.id)
namespace = "#{project.path}-#{project.id}"
expect(kubernetes_namespace).to be_present
expect(kubernetes_namespace.cluster_project_id).to eq(cluster_project.id)
expect(kubernetes_namespace.project_id).to eq(cluster_project.project_id)
expect(kubernetes_namespace.cluster_id).to eq(cluster_project.cluster_id)
expect(kubernetes_namespace.namespace).to eq(namespace)
expect(kubernetes_namespace.service_account_name).to eq("#{namespace}-service-account")
end
end
end
context 'when no Clusters::Project has a Clusters::KubernetesNamespace' do
let(:cluster_projects) { cluster_projects_table.all }
it 'creates a Clusters::KubernetesNamespace per Clusters::Project' do
expect do
migration.perform
end.to change(Clusters::KubernetesNamespace, :count).by(cluster_projects_table.count)
end
it_behaves_like 'consistent kubernetes namespace attributes' do
let(:clusters_with_namespace) { clusters_table.all }
end
end
context 'when every Clusters::Project has Clusters::KubernetesNamespace' do
before do
create_kubernetes_namespace(clusters_table.all)
end
it 'does not create any Clusters::KubernetesNamespace' do
expect do
migration.perform
end.not_to change(Clusters::KubernetesNamespace, :count)
end
end
context 'when only some Clusters::Project have Clusters::KubernetesNamespace related' do
let(:with_kubernetes_namespace) { clusters_table.first(6) }
let(:with_no_kubernetes_namespace) { clusters_table.last(4) }
before do
create_kubernetes_namespace(with_kubernetes_namespace)
end
it 'creates limited number of Clusters::KubernetesNamespace' do
expect do
migration.perform
end.to change(Clusters::KubernetesNamespace, :count).by(with_no_kubernetes_namespace.count)
end
it 'does not modify clusters with Clusters::KubernetesNamespace' do
migration.perform
with_kubernetes_namespace.each do |cluster|
kubernetes_namespace = cluster_kubernetes_namespaces_table.where(cluster_id: cluster.id)
expect(kubernetes_namespace.count).to eq(1)
end
end
it_behaves_like 'consistent kubernetes namespace attributes' do
let(:clusters_with_namespace) { with_no_kubernetes_namespace }
end
end
end
spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let(:todos) { table(:todos) }
let(:issues) { table(:issues) }
let(:assignees) { table(:issue_assignees) }
let(:project_authorizations) { table(:project_authorizations) }
let(:project_features) { table(:project_features) }
let(:todo_params) { { author_id: 1, target_type: 'Issue', action: 1, state: :pending } }
before do
users.create(id: 1, email: 'user@example.com', projects_limit: 10)
users.create(id: 2, email: 'reporter@example.com', projects_limit: 10)
users.create(id: 3, email: 'guest@example.com', projects_limit: 10)
namespace = namespaces.create(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: namespace.id)
projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: namespace.id)
issues.create(id: 1, project_id: 1)
issues.create(id: 2, project_id: 2)
project_authorizations.create(user_id: 2, project_id: 2, access_level: 20) # reporter
project_authorizations.create(user_id: 3, project_id: 2, access_level: 10) # guest
todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 1)) # out of project ids range
todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 2)) # non member
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 2)) # reporter
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 2)) # guest
end
subject { described_class.new.perform(2, 5) }
context 'when a project is private' do
it 'removes todos of users without project access' do
expect { subject }.to change { Todo.count }.from(4).to(3)
end
context 'with a confidential issue' do
it 'removes todos of users without project access and guests for confidential issues' do
issues.create(id: 3, project_id: 2, confidential: true)
issues.create(id: 4, project_id: 1, confidential: true) # not in the batch
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3))
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3))
todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 4))
expect { subject }.to change { Todo.count }.from(7).to(5)
end
end
end
context 'when a project is public' do
before do
projects.find(2).update_attribute(:visibility_level, 20)
end
context 'when all features have the same visibility as the project, no confidential issues' do
it 'does not remove any todos' do
expect { subject }.not_to change { Todo.count }
end
end
context 'with confidential issues' do
before do
users.create(id: 4, email: 'author@example.com', projects_limit: 10)
users.create(id: 5, email: 'assignee@example.com', projects_limit: 10)
issues.create(id: 3, project_id: 2, confidential: true, author_id: 4)
assignees.create(user_id: 5, issue_id: 3)
todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 3)) # to be deleted
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) # authorized user
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) # to be deleted guest
todos.create(todo_params.merge(user_id: 4, project_id: 2, target_id: 3)) # conf issue author
todos.create(todo_params.merge(user_id: 5, project_id: 2, target_id: 3)) # conf issue assignee
end
it 'removes confidential issue todos for non authorized users' do
expect { subject }.to change { Todo.count }.from(9).to(7)
end
end
context 'features visibility restrictions' do
before do
todo_params.merge!(project_id: 2, user_id: 1, target_id: 3)
todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'MergeRequest'))
todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'Commit'))
end
context 'when issues are restricted to project members' do
before do
project_features.create(issues_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
expect { subject }.to change { Todo.count }.from(6).to(5)
end
end
context 'when merge requests are restricted to project members' do
before do
project_features.create(merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
expect { subject }.to change { Todo.count }.from(6).to(5)
end
end
context 'when repository and merge requests are restricted to project members' do
before do
project_features.create(repository_access_level: 10, merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members commit and merge requests todos' do
expect { subject }.to change { Todo.count }.from(6).to(4)
end
end
end
end
end
spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices do
let(:services) { table(:services) }
describe '#perform' do
it 'migrates services where note_events is true' do
service = services.create(confidential_note_events: nil, note_events: true)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(true)
end
it 'ignores services where note_events is false' do
service = services.create(confidential_note_events: nil, note_events: false)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(nil)
end
it 'ignores services where confidential_note_events has already been set' do
service = services.create(confidential_note_events: false, note_events: true)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(false)
end
end
end
spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb deleted 100644 → 0
View file @ 9b491d6a
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks do
let(:web_hooks) { table(:web_hooks) }
describe '#perform' do
it 'migrates hooks where note_events is true' do
hook = web_hooks.create(confidential_note_events: nil, note_events: true)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(true)
end
it 'ignores hooks where note_events is false' do
hook = web_hooks.create(confidential_note_events: nil, note_events: false)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(nil)
end
it 'ignores hooks where confidential_note_events has already been set' do
hook = web_hooks.create(confidential_note_events: false, note_events: true)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(false)
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7