Skip to content

G
gitlab-ce
Commit b8aedf35 authored by Mayra Cabrera's avatar Mayra Cabrera
Browse files
Options

Merge branch 'ab/cleanup-background-migrations' into 'master' 

Delete unused background migration code

Closes #218998

See merge request gitlab-org/gitlab!37423
parents ed445fd6 f36ab2c2
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1516 deletions
lib/gitlab/background_migration/archive_legacy_traces.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class ArchiveLegacyTraces
def perform(start_id, stop_id)
# This background migration directly refers to ::Ci::Build model which is defined in application code.
# In general, migration code should be isolated as much as possible in order to be idempotent.
# However, `archive!` method is too complicated to be replicated by coping its subsequent code.
# So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1
::Ci::Build.finished.without_archived_trace
.where(id: start_id..stop_id).find_each do |build|
build.trace.archive!
rescue => e
Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger
end
end
end
end
end
lib/gitlab/background_migration/backfill_hashed_project_repositories.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class that will fill the project_repositories table for projects that
# are on hashed storage and an entry is is missing in this table.
class BackfillHashedProjectRepositories < BackfillProjectRepositories
private
def projects
Project.on_hashed_storage
end
end
end
end
lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This module is used to write the full path of all projects to
# the git repository config file.
# Storing the full project path in the git config allows admins to
# easily identify a project when it is using hashed storage.
module BackfillProjectFullpathInRepoConfig
OrphanedNamespaceError = Class.new(StandardError)
module Storage
# Class that returns the disk path for a project using hashed storage
class Hashed
attr_accessor :project
ROOT_PATH_PREFIX = '@hashed'
def initialize(project)
@project = project
end
def disk_path
"#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
end
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
end
end
# Class that returns the disk path for a project using legacy storage
class LegacyProject
attr_accessor :project
def initialize(project)
@project = project
end
def disk_path
project.full_path
end
end
end
# Concern used by Project and Namespace to determine the full
# route to the project
module Routable
extend ActiveSupport::Concern
def full_path
@full_path ||= build_full_path
end
def build_full_path
return path unless has_parent?
raise OrphanedNamespaceError if parent.nil?
parent.full_path + '/' + path
end
def has_parent?
read_attribute(association(:parent).reflection.foreign_key)
end
end
# Class used to interact with repository using Gitaly
class Repository
attr_reader :storage
def initialize(storage, relative_path)
@storage = storage
@relative_path = relative_path
end
def gitaly_repository
Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
end
end
# Namespace can be a user or group. It can be the root or a
# child of another namespace.
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = nil
include Routable
belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
has_many :projects, inverse_of: :parent
has_many :namespaces, inverse_of: :parent
end
# Project is where the repository (etc.) is stored
class Project < ActiveRecord::Base
self.table_name = 'projects'
include Routable
include EachBatch
FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
delegate :disk_path, to: :storage
def add_fullpath_config
entries = { FULLPATH_CONFIG_KEY => full_path }
repository_service.set_config(entries)
end
def remove_fullpath_config
repository_service.delete_config([FULLPATH_CONFIG_KEY])
end
def cleanup_repository
repository_service.cleanup
end
def storage
@storage ||=
if hashed_storage?
Storage::Hashed.new(self)
else
Storage::LegacyProject.new(self)
end
end
def hashed_storage?
self.storage_version && self.storage_version >= 1
end
def repository
@repository ||= Repository.new(repository_storage, disk_path + '.git')
end
def repository_service
@repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
end
end
# Base class for Up and Down migration classes
class BackfillFullpathMigration
RETRY_DELAY = 15.minutes
MAX_RETRIES = 2
# Base class for retrying one project
class BaseRetryOne
def perform(project_id, retry_count)
project = Project.find(project_id)
return unless project
migration_class.new.safe_perform_one(project, retry_count)
end
end
def perform(start_id, end_id)
Project.includes(:parent).where(id: start_id..end_id).each do |project|
safe_perform_one(project)
end
end
def safe_perform_one(project, retry_count = 0)
perform_one(project)
rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError
nil
rescue GRPC::BadStatus
schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES
end
def schedule_retry(project, retry_count)
# Constants provided to BackgroundMigrationWorker must be within the
# scope of Gitlab::BackgroundMigration
retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '')
BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count])
end
end
# Class to add the fullpath to the git repo config
class Up < BackfillFullpathMigration
# Class used to retry
class RetryOne < BaseRetryOne
def migration_class
Up
end
end
def perform_one(project)
project.cleanup_repository
project.add_fullpath_config
end
end
# Class to rollback adding the fullpath to the git repo config
class Down < BackfillFullpathMigration
# Class used to retry
class RetryOne < BaseRetryOne
def migration_class
Down
end
end
def perform_one(project)
project.cleanup_repository
project.remove_fullpath_config
end
end
end
end
end
lib/gitlab/background_migration/fill_file_store_job_artifact.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillFileStoreJobArtifact
class JobArtifact < ActiveRecord::Base
self.table_name = 'ci_job_artifacts'
end
def perform(start_id, stop_id)
FillFileStoreJobArtifact::JobArtifact
.where(file_store: nil)
.where(id: (start_id..stop_id))
.update_all(file_store: 1)
end
end
end
end
lib/gitlab/background_migration/fill_file_store_lfs_object.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillFileStoreLfsObject
class LfsObject < ActiveRecord::Base
self.table_name = 'lfs_objects'
end
def perform(start_id, stop_id)
FillFileStoreLfsObject::LfsObject
.where(file_store: nil)
.where(id: (start_id..stop_id))
.update_all(file_store: 1)
end
end
end
end
lib/gitlab/background_migration/fill_store_upload.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FillStoreUpload
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
self.inheritance_column = :_type_disabled
end
def perform(start_id, stop_id)
FillStoreUpload::Upload
.where(store: nil)
.where(id: (start_id..stop_id))
.update_all(store: 1)
end
end
end
end
lib/gitlab/background_migration/fix_cross_project_label_links.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class FixCrossProjectLabelLinks
GROUP_NESTED_LEVEL = 10.freeze
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
class Label < ActiveRecord::Base
self.inheritance_column = :_type_disabled
self.table_name = 'labels'
end
class LabelLink < ActiveRecord::Base
self.table_name = 'label_links'
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
class Namespace < ActiveRecord::Base
self.inheritance_column = :_type_disabled
self.table_name = 'namespaces'
def self.groups_with_descendants_ids(start_id, stop_id)
# To isolate migration code, we avoid usage of
# Gitlab::GroupHierarchy#base_and_descendants which already
# does this job better
ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id)
group_ids = ids
GROUP_NESTED_LEVEL.times do
ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id)
break if ids.empty?
group_ids += ids
end
group_ids.uniq
end
end
def perform(start_id, stop_id)
group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id)
project_ids = Project.where(namespace_id: group_ids).select(:id)
fix_issues(project_ids)
fix_merge_requests(project_ids)
end
private
# select IDs of issues which reference a label which is:
# a) a project label of a different project, or
# b) a group label of a different group than issue's project group
def fix_issues(project_ids)
issue_ids = Label
.joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\'
INNER JOIN issues ON issues.id = label_links.target_id
INNER JOIN projects ON projects.id = issues.project_id')
.where('issues.project_id in (?)', project_ids)
.where('(labels.project_id is not null and labels.project_id != issues.project_id) '\
'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
.select('distinct issues.id')
Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) }
end
# select IDs of MRs which reference a label which is:
# a) a project label of a different project, or
# b) a group label of a different group than MR's project group
def fix_merge_requests(project_ids)
mr_ids = Label
.joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\'
INNER JOIN merge_requests ON merge_requests.id = label_links.target_id
INNER JOIN projects ON projects.id = merge_requests.target_project_id')
.where('merge_requests.target_project_id in (?)', project_ids)
.where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\
'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
.select('distinct merge_requests.id')
MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) }
end
def check_resource_labels(resource, project_id)
local_labels = available_labels(project_id)
# get all label links for the given resource (issue/MR)
# which reference a label not included in available_labels
# (other than its project labels and labels of ancestor groups)
cross_labels = LabelLink
.select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id')
.joins('INNER JOIN labels ON labels.id = label_links.label_id')
.where(target_type: resource.class.name.demodulize, target_id: resource.id)
.where('labels.id not in (?)', local_labels.select(:id))
cross_labels.each do |label|
matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color}
next unless matching_label
Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger
LabelLink.update(label.label_link_id, label_id: matching_label.id)
end
end
# get all labels available for the project (including
# group labels of ancestor groups)
def available_labels(project_id)
@labels ||= {}
@labels[project_id] ||= Label
.where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)",
project_group_ids(project_id),
project_id)
end
def project_group_ids(project_id)
ids = [Project.find(project_id).namespace_id]
GROUP_NESTED_LEVEL.times do
group = Namespace.find(ids.last)
break unless group.parent_id
ids << group.parent_id
end
ids
end
end
end
end
lib/gitlab/background_migration/migrate_build_stage.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateBuildStage
module Migratable
class Stage < ActiveRecord::Base
self.table_name = 'ci_stages'
end
class Build < ActiveRecord::Base
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
def ensure_stage!(attempts: 2)
find_stage || create_stage!
rescue ActiveRecord::RecordNotUnique
retry if (attempts -= 1) > 0
raise
end
def find_stage
Stage.find_by(name: self.stage || 'test',
pipeline_id: self.commit_id,
project_id: self.project_id)
end
def create_stage!
Stage.create!(name: self.stage || 'test',
pipeline_id: self.commit_id,
project_id: self.project_id)
end
end
end
def perform(start_id, stop_id)
stages = Migratable::Build.where('stage_id IS NULL')
.where('id BETWEEN ? AND ?', start_id, stop_id)
.map { |build| build.ensure_stage! }
.compact.map(&:id)
MigrateBuildStageIdReference.new.perform(start_id, stop_id)
MigrateStageStatus.new.perform(stages.min, stages.max)
end
end
end
end
lib/gitlab/background_migration/migrate_build_stage_id_reference.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateBuildStageIdReference
def perform(start_id, stop_id)
sql = <<-SQL.strip_heredoc
UPDATE ci_builds
SET stage_id =
(SELECT id FROM ci_stages
WHERE ci_stages.pipeline_id = ci_builds.commit_id
AND ci_stages.name = ci_builds.stage)
WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
AND ci_builds.stage_id IS NULL
SQL
ActiveRecord::Base.connection.execute(sql)
end
end
end
end
lib/gitlab/background_migration/migrate_stage_index.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class MigrateStageIndex
def perform(start_id, stop_id)
migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql|
ActiveRecord::Base.connection.execute(sql)
end
end
private
def migrate_stage_index_sql(start_id, stop_id)
<<~SQL
WITH freqs AS (
SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds
WHERE stage_id BETWEEN #{start_id} AND #{stop_id}
AND stage_idx IS NOT NULL
GROUP BY stage_id, stage_idx
), indexes AS (
SELECT DISTINCT stage_id, first_value(stage_idx)
OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index
FROM freqs
)
UPDATE ci_stages SET position = indexes.index
FROM indexes WHERE indexes.stage_id = ci_stages.id
AND ci_stages.position IS NULL;
SQL
end
end
end
end
lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
#
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class PopulateClusterKubernetesNamespaceTable
include Gitlab::Database::MigrationHelpers
BATCH_SIZE = 1_000
module Migratable
class KubernetesNamespace < ActiveRecord::Base
self.table_name = 'clusters_kubernetes_namespaces'
end
class ClusterProject < ActiveRecord::Base
include EachBatch
self.table_name = 'cluster_projects'
belongs_to :project
def self.with_no_kubernetes_namespace
where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id))
end
def namespace
slug = "#{project.path}-#{project.id}".downcase
slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
end
def service_account
"#{namespace}-service-account"
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
end
def perform
cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index|
sql_values = sql_values_for(cluster_projects_batch)
insert_into_cluster_kubernetes_namespace(sql_values)
end
end
private
def cluster_projects_with_no_kubernetes_namespace
Migratable::ClusterProject.with_no_kubernetes_namespace
end
def sql_values_for(cluster_projects)
cluster_projects.map do |cluster_project|
values_for_cluster_project(cluster_project)
end
end
def values_for_cluster_project(cluster_project)
{
cluster_project_id: cluster_project.id,
cluster_id: cluster_project.cluster_id,
project_id: cluster_project.project_id,
namespace: cluster_project.namespace,
service_account_name: cluster_project.service_account,
created_at: 'NOW()',
updated_at: 'NOW()'
}
end
def insert_into_cluster_kubernetes_namespace(rows)
Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert
rows,
disable_quote: [:created_at, :updated_at])
end
end
end
end
lib/gitlab/background_migration/remove_restricted_todos.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
# rubocop:disable Metrics/ClassLength
module Gitlab
module BackgroundMigration
class RemoveRestrictedTodos
PRIVATE_FEATURE = 10
PRIVATE_PROJECT = 0
class Project < ActiveRecord::Base
self.table_name = 'projects'
end
class ProjectAuthorization < ActiveRecord::Base
self.table_name = 'project_authorizations'
end
class ProjectFeature < ActiveRecord::Base
self.table_name = 'project_features'
end
class Todo < ActiveRecord::Base
include EachBatch
self.table_name = 'todos'
end
class Issue < ActiveRecord::Base
include EachBatch
self.table_name = 'issues'
end
def perform(start_id, stop_id)
projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)')
.where(id: start_id..stop_id)
projects.each do |project|
remove_confidential_issue_todos(project.id)
if project.visibility_level == PRIVATE_PROJECT
remove_non_members_todos(project.id)
else
remove_restricted_features_todos(project.id)
end
end
end
private
def remove_non_members_todos(project_id)
batch_remove_todos_cte(project_id)
end
def remove_confidential_issue_todos(project_id)
# min access level to access a confidential issue is reporter
min_reporters = authorized_users(project_id)
.select(:user_id)
.where('access_level >= ?', 20)
confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id)
confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch|
batch.each do |issue|
assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id)
todos = Todo.where(target_type: 'Issue', target_id: issue.id)
.where('user_id NOT IN (?)', min_reporters)
.where('user_id NOT IN (?)', assigned_users)
todos = todos.where('user_id != ?', issue.author_id) if issue.author_id
todos.delete_all
end
end
end
def remove_restricted_features_todos(project_id)
ProjectFeature.where(project_id: project_id).each do |project_features|
target_types = []
target_types << 'Issue' if private?(project_features.issues_access_level)
target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level)
target_types << 'Commit' if private?(project_features.repository_access_level)
next if target_types.empty?
batch_remove_todos_cte(project_id, target_types)
end
end
def private?(feature_level)
feature_level == PRIVATE_FEATURE
end
def authorized_users(project_id)
ProjectAuthorization.select(:user_id).where(project_id: project_id)
end
def unauthorized_project_todos(project_id)
Todo.where(project_id: project_id)
.where('user_id NOT IN (?)', authorized_users(project_id))
end
def batch_remove_todos_cte(project_id, target_types = nil)
loop do
count = remove_todos_cte(project_id, target_types)
break if count == 0
end
end
def remove_todos_cte(project_id, target_types = nil)
sql = []
sql << with_all_todos_sql(project_id, target_types)
sql << as_deleted_sql
sql << "SELECT count(*) FROM deleted"
result = Todo.connection.exec_query(sql.join(' '))
result.rows[0][0].to_i
end
def with_all_todos_sql(project_id, target_types = nil)
if target_types
table = Arel::Table.new(:todos)
in_target = table[:target_type].in(target_types)
target_types_sql = " AND #{in_target.to_sql}"
end
<<-SQL
WITH all_todos AS (
SELECT id
FROM "todos"
WHERE "todos"."project_id" = #{project_id}
AND (user_id NOT IN (
SELECT "project_authorizations"."user_id"
FROM "project_authorizations"
WHERE "project_authorizations"."project_id" = #{project_id})
#{target_types_sql}
)
),
SQL
end
def as_deleted_sql
<<-SQL
deleted AS (
DELETE FROM todos
WHERE id IN (
SELECT id
FROM all_todos
LIMIT 5000
)
RETURNING id
)
SQL
end
end
end
end
lib/gitlab/background_migration/set_confidential_note_events_on_services.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
# Ensures services which previously received all notes events continue
# to receive confidential ones.
class SetConfidentialNoteEventsOnServices
class Service < ActiveRecord::Base
self.table_name = 'services'
include ::EachBatch
def self.services_to_update
where(confidential_note_events: nil, note_events: true)
end
end
def perform(start_id, stop_id)
Service.services_to_update
.where(id: start_id..stop_id)
.update_all(confidential_note_events: true)
end
end
end
end
lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
# Ensures hooks which previously received all notes events continue
# to receive confidential ones.
class SetConfidentialNoteEventsOnWebhooks
class WebHook < ActiveRecord::Base
self.table_name = 'web_hooks'
include ::EachBatch
def self.hooks_to_update
where(confidential_note_events: nil, note_events: true)
end
end
def perform(start_id, stop_id)
WebHook.hooks_to_update
.where(id: start_id..stop_id)
.update_all(confidential_note_events: true)
end
end
end
end
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces do
include TraceHelpers
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:builds) { table(:ci_builds) }
let(:job_artifacts) { table(:ci_job_artifacts) }
before do
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
@build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build')
end
context 'when trace file exsits at the right place' do
before do
create_legacy_trace(@build, 'trace in file')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(File.exist?(legacy_trace_path(@build))).to be_truthy
described_class.new.perform(1, 1)
expect(job_artifacts.count).to eq(1)
expect(File.exist?(legacy_trace_path(@build))).to be_falsy
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file')
end
end
context 'when trace file does not exsits at the right place' do
it 'does not raise errors nor create job artifact' do
expect { described_class.new.perform(1, 1) }.not_to raise_error
expect(job_artifacts.count).to eq(0)
end
end
context 'when trace data exsits in database' do
before do
create_legacy_trace_in_db(@build, 'trace in db')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(@build.read_attribute(:trace)).not_to be_empty
described_class.new.perform(1, 1)
@build.reload
expect(job_artifacts.count).to eq(1)
expect(@build.read_attribute(:trace)).to be_nil
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db')
end
end
end
spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories do
it_behaves_like 'backfill migration for project repositories', :hashed
end
spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
let(:subgroup) { namespaces.create!(name: 'bar', path: 'bar', parent_id: group.id) }
describe described_class::Storage::Hashed do
let(:project) { double(id: 555) }
subject(:project_storage) { described_class.new(project) }
it 'has the correct disk_path' do
expect(project_storage.disk_path).to eq('@hashed/91/a7/91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8')
end
end
describe described_class::Storage::LegacyProject do
let(:project) { double(full_path: 'this/is/the/full/path') }
subject(:project_storage) { described_class.new(project) }
it 'has the correct disk_path' do
expect(project_storage.disk_path).to eq('this/is/the/full/path')
end
end
describe described_class::Project do
let(:project_record) { projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') }
subject(:project) { described_class.find(project_record.id) }
describe '#full_path' do
it 'returns path containing all parent namespaces' do
expect(project.full_path).to eq('foo/bar/baz')
end
it 'raises OrphanedNamespaceError when any parent namespace does not exist' do
subgroup.update_attribute(:parent_id, non_existing_record_id)
expect { project.full_path }.to raise_error(Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig::OrphanedNamespaceError)
end
end
end
describe described_class::Up do
describe '#perform' do
subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) }
it 'asks the gitaly client to set config' do
projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz')
projects.create!(namespace_id: subgroup.id, name: 'buzz', path: 'buzz', storage_version: 1)
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/baz')
end
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/buzz')
end
migrate
end
end
end
describe described_class::Down do
describe '#perform' do
subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) }
it 'asks the gitaly client to set config' do
projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz')
expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service|
allow(repository_service).to receive(:cleanup)
expect(repository_service).to receive(:delete_config).with(['gitlab.fullpath'])
end
migrate
end
end
end
end
spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks do
let(:namespaces_table) { table(:namespaces) }
let(:projects_table) { table(:projects) }
let(:issues_table) { table(:issues) }
let(:merge_requests_table) { table(:merge_requests) }
let(:labels_table) { table(:labels) }
let(:label_links_table) { table(:label_links) }
let!(:group1) { namespaces_table.create(id: 10, type: 'Group', name: 'group1', path: 'group1') }
let!(:group2) { namespaces_table.create(id: 20, type: 'Group', name: 'group2', path: 'group2') }
let!(:project1) { projects_table.create(id: 1, name: 'project1', path: 'group1/project1', namespace_id: 10) }
let!(:project2) { projects_table.create(id: 3, name: 'project2', path: 'group1/project2', namespace_id: 20) }
let!(:label1) { labels_table.create(id: 1, title: 'bug', color: 'red', group_id: 10, type: 'GroupLabel') }
let!(:label2) { labels_table.create(id: 2, title: 'bug', color: 'red', group_id: 20, type: 'GroupLabel') }
def create_merge_request(id, project_id)
merge_requests_table.create(id: id,
target_project_id: project_id,
target_branch: 'master',
source_project_id: project_id,
source_branch: 'mr name',
title: "mr name#{id}")
end
def create_issue(id, project_id)
issues_table.create(id: id, title: "issue#{id}", project_id: project_id)
end
def create_resource(target_type, id, project_id)
target_type == 'Issue' ? create_issue(id, project_id) : create_merge_request(id, project_id)
end
shared_examples_for 'resource with cross-project labels' do
it 'updates only cross-project label links which exist in the local project or group' do
create_resource(target_type, 1, 1)
create_resource(target_type, 2, 3)
labels_table.create(id: 3, title: 'bug', color: 'red', project_id: 3, type: 'ProjectLabel')
link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1)
link2 = label_links_table.create(label_id: 3, target_type: target_type, target_id: 2)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
expect(link2.reload.label_id).to eq(3)
end
it 'ignores cross-project label links if label color is different' do
labels_table.create(id: 3, title: 'bug', color: 'green', group_id: 20, type: 'GroupLabel')
create_resource(target_type, 1, 1)
link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(3)
end
it 'ignores cross-project label links if label name is different' do
labels_table.create(id: 3, title: 'bug1', color: 'red', group_id: 20, type: 'GroupLabel')
create_resource(target_type, 1, 1)
link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(3)
end
context 'with nested group' do
before do
namespaces_table.create(id: 11, type: 'Group', name: 'subgroup1', path: 'group1/subgroup1', parent_id: 10)
projects_table.create(id: 2, name: 'subproject1', path: 'group1/subgroup1/subproject1', namespace_id: 11)
create_resource(target_type, 1, 2)
end
it 'ignores label links referencing ancestor group labels' do
labels_table.create(id: 4, title: 'bug', color: 'red', project_id: 2, type: 'ProjectLabel')
label_links_table.create(label_id: 4, target_type: target_type, target_id: 1)
link = label_links_table.create(label_id: 1, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
end
it 'checks also issues and MRs in subgroups' do
link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1)
subject.perform(1, 100)
expect(link.reload.label_id).to eq(1)
end
end
end
context 'resource is Issue' do
it_behaves_like 'resource with cross-project labels' do
let(:target_type) { 'Issue' }
end
end
context 'resource is Merge Request' do
it_behaves_like 'resource with cross-project labels' do
let(:target_type) { 'MergeRequest' }
end
end
end
spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
let(:jobs) { table(:ci_builds) }
let(:statuses) do
{
created: 0,
pending: 1,
running: 2,
success: 3,
failed: 4,
canceled: 5,
skipped: 6,
manual: 7
}
end
before do
namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id)
pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
jobs.create!(id: 1, commit_id: 1, project_id: 123,
stage_idx: 2, stage: 'build', status: :success)
jobs.create!(id: 2, commit_id: 1, project_id: 123,
stage_idx: 2, stage: 'build', status: :success)
jobs.create!(id: 3, commit_id: 1, project_id: 123,
stage_idx: 1, stage: 'test', status: :failed)
jobs.create!(id: 4, commit_id: 1, project_id: 123,
stage_idx: 1, stage: 'test', status: :success)
jobs.create!(id: 5, commit_id: 1, project_id: 123,
stage_idx: 3, stage: 'deploy', status: :pending)
jobs.create!(id: 6, commit_id: 1, project_id: 123,
stage_idx: 3, stage: nil, status: :pending)
end
it 'correctly migrates builds stages' do
expect(stages.count).to be_zero
described_class.new.perform(1, 6)
expect(stages.count).to eq 3
expect(stages.all.pluck(:name)).to match_array %w[test build deploy]
expect(jobs.where(stage_id: nil)).to be_one
expect(jobs.find_by(stage_id: nil).id).to eq 6
expect(stages.all.pluck(:status)).to match_array [statuses[:success],
statuses[:failed],
statuses[:pending]]
end
it 'recovers from unique constraint violation only twice', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/28128' do
allow(described_class::Migratable::Stage)
.to receive(:find_by).and_return(nil)
expect(described_class::Migratable::Stage)
.to receive(:find_by).exactly(3).times
expect { described_class.new.perform(1, 6) }
.to raise_error ActiveRecord::RecordNotUnique
end
context 'when invalid class can be loaded due to single table inheritance' do
let(:commit_status) do
jobs.create!(id: 7, commit_id: 1, project_id: 123, stage_idx: 4,
stage: 'post-deploy', status: :failed)
end
before do
commit_status.update_column(:type, 'SomeClass')
end
it 'does ignore single table inheritance type' do
expect { described_class.new.perform(1, 7) }.not_to raise_error
expect(jobs.find(7)).to have_attributes(stage_id: (a_value > 0))
end
end
end
spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:stages) { table(:ci_stages) }
let(:jobs) { table(:ci_builds) }
let(:namespace) { namespaces.create(name: 'gitlab-org', path: 'gitlab-org') }
let(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') }
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
let(:stage1) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'build') }
let(:stage2) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'test') }
before do
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 2, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 10, stage_id: stage1.id)
jobs.create!(commit_id: pipeline.id, project_id: project.id,
stage_idx: 3, stage_id: stage2.id)
end
it 'correctly migrates stages indices' do
expect(stages.all.pluck(:position)).to all(be_nil)
described_class.new.perform(stage1.id, stage2.id)
expect(stages.all.order(:id).pluck(:position)).to eq [2, 3]
end
end
spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable do
include MigrationHelpers::ClusterHelpers
let(:migration) { described_class.new }
let(:clusters_table) { table(:clusters) }
let(:cluster_projects_table) { table(:cluster_projects) }
let(:cluster_kubernetes_namespaces_table) { table(:clusters_kubernetes_namespaces) }
let(:projects_table) { table(:projects) }
let(:namespaces_table) { table(:namespaces) }
let(:provider_gcp_table) { table(:cluster_providers_gcp) }
let(:platform_kubernetes_table) { table(:cluster_platforms_kubernetes) }
before do
create_cluster_project_list(10)
end
shared_examples 'consistent kubernetes namespace attributes' do
it 'populates namespace and service account information' do
migration.perform
clusters_with_namespace.each do |cluster|
cluster_project = cluster_projects_table.find_by(cluster_id: cluster.id)
project = projects_table.find(cluster_project.project_id)
kubernetes_namespace = cluster_kubernetes_namespaces_table.find_by(cluster_id: cluster.id)
namespace = "#{project.path}-#{project.id}"
expect(kubernetes_namespace).to be_present
expect(kubernetes_namespace.cluster_project_id).to eq(cluster_project.id)
expect(kubernetes_namespace.project_id).to eq(cluster_project.project_id)
expect(kubernetes_namespace.cluster_id).to eq(cluster_project.cluster_id)
expect(kubernetes_namespace.namespace).to eq(namespace)
expect(kubernetes_namespace.service_account_name).to eq("#{namespace}-service-account")
end
end
end
context 'when no Clusters::Project has a Clusters::KubernetesNamespace' do
let(:cluster_projects) { cluster_projects_table.all }
it 'creates a Clusters::KubernetesNamespace per Clusters::Project' do
expect do
migration.perform
end.to change(Clusters::KubernetesNamespace, :count).by(cluster_projects_table.count)
end
it_behaves_like 'consistent kubernetes namespace attributes' do
let(:clusters_with_namespace) { clusters_table.all }
end
end
context 'when every Clusters::Project has Clusters::KubernetesNamespace' do
before do
create_kubernetes_namespace(clusters_table.all)
end
it 'does not create any Clusters::KubernetesNamespace' do
expect do
migration.perform
end.not_to change(Clusters::KubernetesNamespace, :count)
end
end
context 'when only some Clusters::Project have Clusters::KubernetesNamespace related' do
let(:with_kubernetes_namespace) { clusters_table.first(6) }
let(:with_no_kubernetes_namespace) { clusters_table.last(4) }
before do
create_kubernetes_namespace(with_kubernetes_namespace)
end
it 'creates limited number of Clusters::KubernetesNamespace' do
expect do
migration.perform
end.to change(Clusters::KubernetesNamespace, :count).by(with_no_kubernetes_namespace.count)
end
it 'does not modify clusters with Clusters::KubernetesNamespace' do
migration.perform
with_kubernetes_namespace.each do |cluster|
kubernetes_namespace = cluster_kubernetes_namespaces_table.where(cluster_id: cluster.id)
expect(kubernetes_namespace.count).to eq(1)
end
end
it_behaves_like 'consistent kubernetes namespace attributes' do
let(:clusters_with_namespace) { with_no_kubernetes_namespace }
end
end
end
spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:users) { table(:users) }
let(:todos) { table(:todos) }
let(:issues) { table(:issues) }
let(:assignees) { table(:issue_assignees) }
let(:project_authorizations) { table(:project_authorizations) }
let(:project_features) { table(:project_features) }
let(:todo_params) { { author_id: 1, target_type: 'Issue', action: 1, state: :pending } }
before do
users.create(id: 1, email: 'user@example.com', projects_limit: 10)
users.create(id: 2, email: 'reporter@example.com', projects_limit: 10)
users.create(id: 3, email: 'guest@example.com', projects_limit: 10)
namespace = namespaces.create(name: 'gitlab-org', path: 'gitlab-org')
projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: namespace.id)
projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: namespace.id)
issues.create(id: 1, project_id: 1)
issues.create(id: 2, project_id: 2)
project_authorizations.create(user_id: 2, project_id: 2, access_level: 20) # reporter
project_authorizations.create(user_id: 3, project_id: 2, access_level: 10) # guest
todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 1)) # out of project ids range
todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 2)) # non member
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 2)) # reporter
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 2)) # guest
end
subject { described_class.new.perform(2, 5) }
context 'when a project is private' do
it 'removes todos of users without project access' do
expect { subject }.to change { Todo.count }.from(4).to(3)
end
context 'with a confidential issue' do
it 'removes todos of users without project access and guests for confidential issues' do
issues.create(id: 3, project_id: 2, confidential: true)
issues.create(id: 4, project_id: 1, confidential: true) # not in the batch
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3))
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3))
todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 4))
expect { subject }.to change { Todo.count }.from(7).to(5)
end
end
end
context 'when a project is public' do
before do
projects.find(2).update_attribute(:visibility_level, 20)
end
context 'when all features have the same visibility as the project, no confidential issues' do
it 'does not remove any todos' do
expect { subject }.not_to change { Todo.count }
end
end
context 'with confidential issues' do
before do
users.create(id: 4, email: 'author@example.com', projects_limit: 10)
users.create(id: 5, email: 'assignee@example.com', projects_limit: 10)
issues.create(id: 3, project_id: 2, confidential: true, author_id: 4)
assignees.create(user_id: 5, issue_id: 3)
todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 3)) # to be deleted
todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) # authorized user
todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) # to be deleted guest
todos.create(todo_params.merge(user_id: 4, project_id: 2, target_id: 3)) # conf issue author
todos.create(todo_params.merge(user_id: 5, project_id: 2, target_id: 3)) # conf issue assignee
end
it 'removes confidential issue todos for non authorized users' do
expect { subject }.to change { Todo.count }.from(9).to(7)
end
end
context 'features visibility restrictions' do
before do
todo_params.merge!(project_id: 2, user_id: 1, target_id: 3)
todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'MergeRequest'))
todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'Commit'))
end
context 'when issues are restricted to project members' do
before do
project_features.create(issues_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
expect { subject }.to change { Todo.count }.from(6).to(5)
end
end
context 'when merge requests are restricted to project members' do
before do
project_features.create(merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members issue todos' do
expect { subject }.to change { Todo.count }.from(6).to(5)
end
end
context 'when repository and merge requests are restricted to project members' do
before do
project_features.create(repository_access_level: 10, merge_requests_access_level: 10, pages_access_level: 10, project_id: 2)
end
it 'removes non members commit and merge requests todos' do
expect { subject }.to change { Todo.count }.from(6).to(4)
end
end
end
end
end
spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices do
let(:services) { table(:services) }
describe '#perform' do
it 'migrates services where note_events is true' do
service = services.create(confidential_note_events: nil, note_events: true)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(true)
end
it 'ignores services where note_events is false' do
service = services.create(confidential_note_events: nil, note_events: false)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(nil)
end
it 'ignores services where confidential_note_events has already been set' do
service = services.create(confidential_note_events: false, note_events: true)
subject.perform(service.id, service.id)
expect(service.reload.confidential_note_events).to eq(false)
end
end
end
spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb deleted 100644 → 0
View file @ ed445fd6
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks do
let(:web_hooks) { table(:web_hooks) }
describe '#perform' do
it 'migrates hooks where note_events is true' do
hook = web_hooks.create(confidential_note_events: nil, note_events: true)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(true)
end
it 'ignores hooks where note_events is false' do
hook = web_hooks.create(confidential_note_events: nil, note_events: false)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(nil)
end
it 'ignores hooks where confidential_note_events has already been set' do
hook = web_hooks.create(confidential_note_events: false, note_events: true)
subject.perform(hook.id, hook.id)
expect(hook.reload.confidential_note_events).to eq(false)
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7