Commit ee79ee94 authored by Shinya Maeda's avatar Shinya Maeda

Cleanup legacy artifact background migration

Add migration and spec

commit 3cc12e1268a6865f524d8fab1804f018312fdf5a
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jan 8 19:34:31 2019 +0900

    Add changelog to this change

commit 5006fc96e38db514956a35f53ae8ee536548a2e9
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Tue Jan 8 19:32:56 2019 +0900

    Remove partial index from ci_builds artifact_file

    ok

    Update schema

commit 3c956bdc02b195bc67d0327bf3748a631ea5466d
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jan 7 21:41:27 2019 +0900

    Add frozen_string_literal: true

commit 8c827cd616890160e6e8908843403a6f20c03236
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Mon Jan 7 21:40:06 2019 +0900

    Set batch size 100

commit aeee559777d3bdeadfd2b9bb61d460f2dc1fa8a6
Author: Shinya Maeda <shinya@gitlab.com>
Date:   Fri Jan 4 18:33:05 2019 +0900

    Cleanup legacy artifact background migration
parent 7b4ca0c9
---
title: Cleanup legacy artifact background migration
merge_request: 24144
author:
type: other
# frozen_string_literal: true
class CleanupLegacyArtifactMigration < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
class Build < ActiveRecord::Base
include EachBatch
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled
scope :with_legacy_artifacts, -> { where("artifacts_file <> ''") }
end
def up
Gitlab::BackgroundMigration.steal('MigrateLegacyArtifacts')
CleanupLegacyArtifactMigration::Build
.with_legacy_artifacts
.each_batch(of: 100) do |batch|
range = batch.pluck('MIN(id)', 'MAX(id)').first
Gitlab::BackgroundMigration::MigrateLegacyArtifacts.new.perform(*range)
end
end
def down
# no-op
end
end
# frozen_string_literal: true
class RemovePartialIndexFromCiBuildsArtifactsFile < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'partial_index_ci_builds_on_id_with_legacy_artifacts'.freeze
disable_ddl_transaction!
def up
remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
end
def down
add_concurrent_index(:ci_builds, :id, where: "artifacts_file <> ''", name: INDEX_NAME)
end
end
...@@ -354,7 +354,6 @@ ActiveRecord::Schema.define(version: 20190115054216) do ...@@ -354,7 +354,6 @@ ActiveRecord::Schema.define(version: 20190115054216) do
t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
t.index ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree t.index ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
t.index ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree
t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))", using: :btree t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))", using: :btree
t.index ["protected"], name: "index_ci_builds_on_protected", using: :btree t.index ["protected"], name: "index_ci_builds_on_protected", using: :btree
......
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20190104182041_cleanup_legacy_artifact_migration.rb')
describe CleanupLegacyArtifactMigration, :migration, :sidekiq, :redis do
let(:migration) { spy('migration') }
context 'when still legacy artifacts exist' do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:pipelines) { table(:ci_pipelines) }
let(:jobs) { table(:ci_builds) }
let(:job_artifacts) { table(:ci_job_artifacts) }
let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
let(:project) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
let(:archive_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::ARCHIVE_FILE_TYPE }
let(:metadata_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::METADATA_FILE_TYPE }
let(:local_store) { ::ObjectStorage::Store::LOCAL }
let(:remote_store) { ::ObjectStorage::Store::REMOTE }
let(:legacy_location) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::LEGACY_PATH_FILE_LOCATION }
before do
jobs.create!(id: 1, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip')
jobs.create!(id: 2, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_metadata: 'metadata.gz')
jobs.create!(id: 3, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
jobs.create!(id: 4, commit_id: pipeline.id, project_id: project.id, status: :running)
jobs.create!(id: 5, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip', artifacts_file_store: remote_store, artifacts_metadata: 'metadata.gz')
jobs.create!(id: 6, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
end
it 'steals sidekiq jobs from MigrateLegacyArtifacts background migration' do
expect(Gitlab::BackgroundMigration).to receive(:steal).with('MigrateLegacyArtifacts')
migrate!
end
it 'migrates legacy artifacts to ci_job_artifacts table' do
migrate!
expect(job_artifacts.order(:job_id, :file_type).pluck('project_id, job_id, file_type, file_store, size, expire_at, file, file_sha256, file_location'))
.to eq([[project.id, 1, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
[project.id, 3, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
[project.id, 3, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
[project.id, 5, archive_file_type, remote_store, nil, nil, 'archive.zip', nil, legacy_location],
[project.id, 5, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
[project.id, 6, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
[project.id, 6, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location]])
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment