Commit 2155ba8b authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '51651-fill-pipeline-source-for-external-pipelines' into 'master'

Resolve "Fill pipeline source for external pipelines"

Closes #51651

See merge request gitlab-org/gitlab-ce!21814
parents 227cc997 eac01d4f
---
title: Retroactively fill pipeline source for external pipelines.
merge_request: 21814
author:
type: other
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddIndexPipelinesProjectIdSource < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_pipelines, [:project_id, :source]
end
def down
remove_concurrent_index :ci_pipelines, [:project_id, :source]
end
end
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class PopulateExternalPipelineSource < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
MIGRATION = 'PopulateExternalPipelineSource'.freeze
BATCH_SIZE = 500
disable_ddl_transaction!
class Pipeline < ActiveRecord::Base
include EachBatch
self.table_name = 'ci_pipelines'
end
def up
Pipeline.where(source: nil).tap do |relation|
queue_background_migration_jobs_by_range_at_intervals(relation,
MIGRATION,
5.minutes,
batch_size: BATCH_SIZE)
end
end
def down
# noop
end
end
...@@ -475,6 +475,7 @@ ActiveRecord::Schema.define(version: 20180917172041) do ...@@ -475,6 +475,7 @@ ActiveRecord::Schema.define(version: 20180917172041) do
add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
add_index "ci_pipelines", ["project_id", "source"], name: "index_ci_pipelines_on_project_id_and_source", using: :btree
add_index "ci_pipelines", ["project_id", "status", "config_source"], name: "index_ci_pipelines_on_project_id_and_status_and_config_source", using: :btree add_index "ci_pipelines", ["project_id", "status", "config_source"], name: "index_ci_pipelines_on_project_id_and_status_and_config_source", using: :btree
add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
......
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class PopulateExternalPipelineSource
module Migratable
class Pipeline < ActiveRecord::Base
self.table_name = 'ci_pipelines'
def self.sources
{
unknown: nil,
push: 1,
web: 2,
trigger: 3,
schedule: 4,
api: 5,
external: 6
}
end
end
class CommitStatus < ActiveRecord::Base
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') }
scope :of_type, -> (type) { where('type=?', type) }
end
end
def perform(start_id, stop_id)
external_pipelines(start_id, stop_id)
.update_all(source: Migratable::Pipeline.sources[:external])
end
private
def external_pipelines(start_id, stop_id)
Migratable::Pipeline.where(id: (start_id..stop_id))
.where(
'EXISTS (?) AND NOT EXISTS (?)',
Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1),
Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1)
)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do
let(:migration) { described_class.new }
let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
let(:pipelines) { [internal_pipeline, unknown_pipeline].map(&:id) }
let!(:unknown_pipeline) do
build(:ci_pipeline, source: :unknown)
.tap { |pipeline| pipeline.save(validate: false) }
end
subject { migration.perform(pipelines.min, pipelines.max) }
shared_examples 'no changes' do
it 'does not change the pipeline source' do
expect { subject }.not_to change { unknown_pipeline.reload.source }
end
end
context 'when unknown pipeline is external' do
before do
create(:generic_commit_status, pipeline: unknown_pipeline)
end
it 'populates the pipeline source' do
subject
expect(unknown_pipeline.reload.source).to eq('external')
end
it 'can be repeated without effect' do
subject
expect { subject }.not_to change { unknown_pipeline.reload.source }
end
end
context 'when unknown pipeline has just a build' do
before do
create(:ci_build, pipeline: unknown_pipeline)
end
it_behaves_like 'no changes'
end
context 'when unknown pipeline has no statuses' do
it_behaves_like 'no changes'
end
context 'when unknown pipeline has a build and a status' do
before do
create(:generic_commit_status, pipeline: unknown_pipeline)
create(:ci_build, pipeline: unknown_pipeline)
end
it_behaves_like 'no changes'
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment