Commit acd0b691 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'feature/gb/migrate-pipeline-stages' into 'master'

Migrate pipeline stages in the database

See merge request !11714
parents cedfde54 36ed05fa
class CreatePipelineStages < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
create_table :ci_stages do |t|
t.integer :project_id
t.integer :pipeline_id
t.timestamps null: true
t.string :name
end
add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade
add_concurrent_foreign_key :ci_stages, :ci_pipelines, column: :pipeline_id, on_delete: :cascade
add_concurrent_index :ci_stages, :project_id
add_concurrent_index :ci_stages, :pipeline_id
end
def down
drop_table :ci_stages
end
end
class AddStageIdToCiBuilds < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column :ci_builds, :stage_id, :integer
add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade
add_concurrent_index :ci_builds, :stage_id
end
def down
remove_foreign_key :ci_builds, column: :stage_id
remove_concurrent_index :ci_builds, :stage_id
remove_column :ci_builds, :stage_id, :integer
end
end
class MigratePipelineStages < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
disable_statement_timeout
execute <<-SQL.strip_heredoc
INSERT INTO ci_stages (project_id, pipeline_id, name)
SELECT project_id, commit_id, stage FROM ci_builds
WHERE stage IS NOT NULL
AND stage_id IS NULL
AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id)
AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id)
GROUP BY project_id, commit_id, stage
ORDER BY MAX(stage_idx)
SQL
end
end
class CreateIndexInPipelineStages < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index(:ci_stages, [:pipeline_id, :name])
end
def down
remove_concurrent_index(:ci_stages, [:pipeline_id, :name])
end
end
class MigrateBuildStageReference < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
disable_statement_timeout
stage_id = Arel.sql <<-SQL.strip_heredoc
(SELECT id FROM ci_stages
WHERE ci_stages.pipeline_id = ci_builds.commit_id
AND ci_stages.name = ci_builds.stage)
SQL
update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query|
query.where(table[:stage_id].eq(nil))
end
end
def down
disable_statement_timeout
update_column_in_batches(:ci_builds, :stage_id, nil)
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20170525174156) do ActiveRecord::Schema.define(version: 20170526185921) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -234,6 +234,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do ...@@ -234,6 +234,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do
t.string "coverage_regex" t.string "coverage_regex"
t.integer "auto_canceled_by_id" t.integer "auto_canceled_by_id"
t.boolean "retried" t.boolean "retried"
t.integer "stage_id"
end end
add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
...@@ -243,6 +244,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do ...@@ -243,6 +244,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do
add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree
add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree
add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree
add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree
add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree
...@@ -327,6 +329,18 @@ ActiveRecord::Schema.define(version: 20170525174156) do ...@@ -327,6 +329,18 @@ ActiveRecord::Schema.define(version: 20170525174156) do
add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree
add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree
create_table "ci_stages", force: :cascade do |t|
t.integer "project_id"
t.integer "pipeline_id"
t.datetime "created_at"
t.datetime "updated_at"
t.string "name"
end
add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
add_index "ci_stages", ["pipeline_id"], name: "index_ci_stages_on_pipeline_id", using: :btree
add_index "ci_stages", ["project_id"], name: "index_ci_stages_on_project_id", using: :btree
create_table "ci_trigger_requests", force: :cascade do |t| create_table "ci_trigger_requests", force: :cascade do |t|
t.integer "trigger_id", null: false t.integer "trigger_id", null: false
t.text "variables" t.text "variables"
...@@ -1484,10 +1498,13 @@ ActiveRecord::Schema.define(version: 20170525174156) do ...@@ -1484,10 +1498,13 @@ ActiveRecord::Schema.define(version: 20170525174156) do
add_foreign_key "boards", "projects" add_foreign_key "boards", "projects"
add_foreign_key "chat_teams", "namespaces", on_delete: :cascade add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
......
...@@ -214,6 +214,7 @@ CommitStatus: ...@@ -214,6 +214,7 @@ CommitStatus:
- stage - stage
- trigger_request_id - trigger_request_id
- stage_idx - stage_idx
- stage_id
- tag - tag
- ref - ref
- user_id - user_id
......
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb')
describe MigrateBuildStageReference, :migration do
##
# Create test data - pipeline and CI/CD jobs.
#
let(:jobs) { table(:ci_builds) }
let(:stages) { table(:ci_stages) }
let(:pipelines) { table(:ci_pipelines) }
let(:projects) { table(:projects) }
before do
# Create projects
#
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
# Create CI/CD pipelines
#
pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
# Create CI/CD jobs
#
jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy')
# Create CI/CD stages
#
stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test')
stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build')
stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy')
stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1')
stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2')
stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy')
end
it 'correctly migrate build stage references' do
expect(jobs.where(stage_id: nil).count).to eq 8
migrate!
expect(jobs.where(stage_id: nil).count).to eq 1
expect(jobs.find(1).stage_id).to eq 102
expect(jobs.find(2).stage_id).to eq 102
expect(jobs.find(3).stage_id).to eq 101
expect(jobs.find(4).stage_id).to eq 103
expect(jobs.find(5).stage_id).to eq 105
expect(jobs.find(6).stage_id).to eq 104
expect(jobs.find(7).stage_id).to eq 104
expect(jobs.find(8).stage_id).to eq nil
end
end
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb')
describe MigratePipelineStages, :migration do
##
# Create test data - pipeline and CI/CD jobs.
#
let(:jobs) { table(:ci_builds) }
let(:stages) { table(:ci_stages) }
let(:pipelines) { table(:ci_pipelines) }
let(:projects) { table(:projects) }
before do
# Create projects
#
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
# Create CI/CD pipelines
#
pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
# Create CI/CD jobs
#
jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy')
jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
jobs.create!(id: 11, commit_id: 3, project_id: 456, stage_idx: 3, stage: 'deploy')
jobs.create!(id: 12, commit_id: 2, project_id: 789, stage_idx: 3, stage: 'deploy')
end
it 'correctly migrates pipeline stages' do
expect(stages.count).to be_zero
migrate!
expect(stages.count).to eq 6
expect(stages.all.pluck(:name))
.to match_array %w[test build deploy test:1 test:2 deploy]
expect(stages.where(pipeline_id: 1).order(:id).pluck(:name))
.to eq %w[test build deploy]
expect(stages.where(pipeline_id: 2).order(:id).pluck(:name))
.to eq %w[test:1 test:2 deploy]
expect(stages.where(pipeline_id: 3).count).to be_zero
expect(stages.where(project_id: 789).count).to be_zero
end
end
...@@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do ...@@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do
updated_at started_at finished_at queued_at erased_by updated_at started_at finished_at queued_at erased_by
erased_at auto_canceled_by].freeze erased_at auto_canceled_by].freeze
# TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR.
IGNORE_ACCESSORS = IGNORE_ACCESSORS =
%i[type lock_version target_url base_tags %i[type lock_version target_url base_tags
commit_id deployments erased_by_id last_deployment project_id commit_id deployments erased_by_id last_deployment project_id
runner_id tag_taggings taggings tags trigger_request_id runner_id tag_taggings taggings tags trigger_request_id
user_id auto_canceled_by_id retried].freeze user_id auto_canceled_by_id retried stage_id].freeze
shared_examples 'build duplication' do shared_examples 'build duplication' do
let(:build) do let(:build) do
......
...@@ -56,6 +56,7 @@ RSpec.configure do |config| ...@@ -56,6 +56,7 @@ RSpec.configure do |config|
config.include StubGitlabCalls config.include StubGitlabCalls
config.include StubGitlabData config.include StubGitlabData
config.include ApiHelpers, :api config.include ApiHelpers, :api
config.include MigrationsHelpers, :migration
config.infer_spec_type_from_file_location! config.infer_spec_type_from_file_location!
...@@ -97,6 +98,17 @@ RSpec.configure do |config| ...@@ -97,6 +98,17 @@ RSpec.configure do |config|
Sidekiq.redis(&:flushall) Sidekiq.redis(&:flushall)
end end
config.around(:example, :migration) do |example|
begin
ActiveRecord::Migrator
.migrate(migrations_paths, previous_migration.version)
example.run
ensure
ActiveRecord::Migrator.migrate(migrations_paths)
end
end
config.around(:each, :nested_groups) do |example| config.around(:each, :nested_groups) do |example|
example.run if Group.supports_nested_groups? example.run if Group.supports_nested_groups?
end end
......
...@@ -19,6 +19,10 @@ RSpec.configure do |config| ...@@ -19,6 +19,10 @@ RSpec.configure do |config|
DatabaseCleaner.strategy = :truncation DatabaseCleaner.strategy = :truncation
end end
config.before(:each, :migration) do
DatabaseCleaner.strategy = :truncation
end
config.before(:each) do config.before(:each) do
DatabaseCleaner.start DatabaseCleaner.start
end end
......
module MigrationsHelpers
def table(name)
Class.new(ActiveRecord::Base) { self.table_name = name }
end
def migrations_paths
ActiveRecord::Migrator.migrations_paths
end
def table_exists?(name)
ActiveRecord::Base.connection.table_exists?(name)
end
def migrations
ActiveRecord::Migrator.migrations(migrations_paths)
end
def previous_migration
migrations.each_cons(2) do |previous, migration|
break previous if migration.name == described_class.name
end
end
def migrate!
ActiveRecord::Migrator.up(migrations_paths) do |migration|
migration.name == described_class.name
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment