Skip to content

G
gitlab-ce
Commit 5ac30744 authored by Patrick Bajao's avatar Patrick Bajao
Browse files
Options

Disable LinkLfsObjects background migration 

Some jobs are timing out and failing on staging.gitlab.com. This
also disables the post deploy migration scheduling the background
migration.

https://sentry.gitlab.net/gitlab/staginggitlabcom/issues/1306635/events/latest/
parent 843ca90c
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 188 deletions
db/post_migrate/20200210062432_schedule_link_lfs_objects.rb
View file @ 5ac30744
......@@ -4,32 +4,11 @@ class ScheduleLinkLfsObjects < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
MIGRATION = 'LinkLfsObjects'
BATCH_SIZE = 1_000
disable_ddl_transaction!
class Project < ActiveRecord::Base
include EachBatch
self.table_name = 'projects'
end
def up
fork_network_members =
Gitlab::BackgroundMigration::LinkLfsObjects::ForkNetworkMember
.select(1)
.with_non_existing_lfs_objects
.where('fork_network_members.project_id = projects.id')
forks = Project.where('EXISTS (?)', fork_network_members)
queue_background_migration_jobs_by_range_at_intervals(
forks,
MIGRATION,
BackgroundMigrationWorker.minimum_interval,
batch_size: BATCH_SIZE
)
# no-op as background migration being schedule times out in some instances
end
def down
......
lib/gitlab/background_migration/link_lfs_objects.rb
View file @ 5ac30744
......@@ -24,24 +24,7 @@ module Gitlab
end
def perform(start_id, end_id)
select_query =
ForkNetworkMember
.select('lop.lfs_object_id, fork_network_members.project_id')
.with_non_existing_lfs_objects
.where(project_id: start_id..end_id)
return if select_query.empty?
execute <<-SQL
INSERT INTO lfs_objects_projects (lfs_object_id, project_id)
#{select_query.to_sql}
SQL
end
private
def execute(sql)
::ActiveRecord::Base.connection.execute(sql)
# no-op as some queries times out
end
end
end
......
spec/lib/gitlab/background_migration/link_lfs_objects_spec.rb deleted 100644 → 0
View file @ 843ca90c
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::BackgroundMigration::LinkLfsObjects, :migration, schema: 2020_02_10_062432 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:fork_networks) { table(:fork_networks) }
let(:fork_network_members) { table(:fork_network_members) }
let(:lfs_objects) { table(:lfs_objects) }
let(:lfs_objects_projects) { table(:lfs_objects_projects) }
let(:namespace) { namespaces.create(name: 'GitLab', path: 'gitlab') }
let(:source_project) { projects.create(namespace_id: namespace.id) }
let(:another_source_project) { projects.create(namespace_id: namespace.id) }
let(:project) { projects.create(namespace_id: namespace.id) }
let(:another_project) { projects.create(namespace_id: namespace.id) }
let(:other_project) { projects.create(namespace_id: namespace.id) }
let(:linked_project) { projects.create(namespace_id: namespace.id) }
let(:fork_network) { fork_networks.create(root_project_id: source_project.id) }
let(:another_fork_network) { fork_networks.create(root_project_id: another_source_project.id) }
let(:lfs_object) { lfs_objects.create(oid: 'abc123', size: 100) }
let(:another_lfs_object) { lfs_objects.create(oid: 'def456', size: 200) }
before do
# Create links between projects
fork_network_members.create(fork_network_id: fork_network.id, project_id: source_project.id, forked_from_project_id: nil)
[project, another_project, linked_project].each do |p|
fork_network_members.create(
fork_network_id: fork_network.id,
project_id: p.id,
forked_from_project_id: fork_network.root_project_id
)
end
fork_network_members.create(fork_network_id: another_fork_network.id, project_id: another_source_project.id, forked_from_project_id: nil)
fork_network_members.create(fork_network_id: another_fork_network.id, project_id: other_project.id, forked_from_project_id: another_fork_network.root_project_id)
# Links LFS objects to some projects
[source_project, another_source_project, linked_project].each do |p|
lfs_objects_projects.create(lfs_object_id: lfs_object.id, project_id: p.id)
lfs_objects_projects.create(lfs_object_id: another_lfs_object.id, project_id: p.id)
end
end
it 'creates LfsObjectsProject records for forks within the specified range of project IDs' do
expect { subject.perform(project.id, other_project.id) }.to change { lfs_objects_projects.count }.by(6)
expect(lfs_object_ids_for(project)).to match_array(lfs_object_ids_for(source_project))
expect(lfs_object_ids_for(another_project)).to match_array(lfs_object_ids_for(source_project))
expect(lfs_object_ids_for(other_project)).to match_array(lfs_object_ids_for(another_source_project))
expect { subject.perform(project.id, other_project.id) }.not_to change { lfs_objects_projects.count }
end
context 'when it is not necessary to create LfsObjectProject records' do
it 'does not create LfsObjectProject records' do
expect { subject.perform(linked_project.id, linked_project.id) }
.not_to change { lfs_objects_projects.count }
end
end
def lfs_object_ids_for(project)
lfs_objects_projects.where(project_id: project.id).pluck(:lfs_object_id)
end
end
spec/migrations/schedule_link_lfs_objects_spec.rb deleted 100644 → 0
View file @ 843ca90c
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200210062432_schedule_link_lfs_objects.rb')
describe ScheduleLinkLfsObjects, :migration, :sidekiq do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:fork_networks) { table(:fork_networks) }
let(:fork_network_members) { table(:fork_network_members) }
let(:lfs_objects) { table(:lfs_objects) }
let(:lfs_objects_projects) { table(:lfs_objects_projects) }
let(:namespace) { namespaces.create(name: 'GitLab', path: 'gitlab') }
let(:fork_network) { fork_networks.create(root_project_id: source_project.id) }
let(:another_fork_network) { fork_networks.create(root_project_id: another_source_project.id) }
let(:source_project) { projects.create(namespace_id: namespace.id) }
let(:another_source_project) { projects.create(namespace_id: namespace.id) }
let(:project) { projects.create(namespace_id: namespace.id) }
let(:another_project) { projects.create(namespace_id: namespace.id) }
let(:other_project) { projects.create(namespace_id: namespace.id) }
let(:linked_project) { projects.create(namespace_id: namespace.id) }
let(:lfs_object) { lfs_objects.create(oid: 'abc123', size: 100) }
let(:another_lfs_object) { lfs_objects.create(oid: 'def456', size: 200) }
before do
# Create links between projects
fork_network_members.create(fork_network_id: fork_network.id, project_id: source_project.id, forked_from_project_id: nil)
[project, another_project, linked_project].each do |p|
fork_network_members.create(
fork_network_id: fork_network.id,
project_id: p.id,
forked_from_project_id: fork_network.root_project_id
)
end
fork_network_members.create(fork_network_id: another_fork_network.id, project_id: another_source_project.id, forked_from_project_id: nil)
fork_network_members.create(fork_network_id: another_fork_network.id, project_id: other_project.id, forked_from_project_id: another_fork_network.root_project_id)
end
context 'when there are forks to be backfilled' do
before do
stub_const("#{described_class.name}::BATCH_SIZE", 2)
# Links LFS objects to some projects
[source_project, another_source_project, linked_project].each do |p|
lfs_objects_projects.create(lfs_object_id: lfs_object.id, project_id: p.id)
lfs_objects_projects.create(lfs_object_id: another_lfs_object.id, project_id: p.id)
end
end
it 'schedules background migration to link LFS objects' do
Sidekiq::Testing.fake! do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(2.minutes, project.id, another_project.id)
expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(4.minutes, other_project.id, other_project.id)
end
end
end
context 'when there are no forks to be backfilled' do
before do
# Links LFS objects to all projects
projects.all.each do |p|
lfs_objects_projects.create(lfs_object_id: lfs_object.id, project_id: p.id)
lfs_objects_projects.create(lfs_object_id: another_lfs_object.id, project_id: p.id)
end
end
it 'does not schedule any job' do
Sidekiq::Testing.fake! do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq(0)
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7