Merge branch 'add-background-migration-to-fill-file-store' into 'master'

Add background migration to fill file stores from `NULL`  to `1`

Closes #45337 and #45476

See merge request gitlab-org/gitlab-ce!18557

changelogs/unreleased/add-background-migration-to-fill-file-store.yml

+---
+title: Add backgound migration for filling nullfied file_store columns
+merge_request: 18557
+author:
+type: performance

db/post_migrate/20180424151928_fill_file_store.rb

+class FillFileStore < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_job_artifacts'
+    BATCH_SIZE = 10_000
+
+    def self.params_for_background_migration
+      yield self.where(file_store: nil), 'FillFileStoreJobArtifact', 5.minutes, BATCH_SIZE
+    end
+  end
+
+  class LfsObject < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'lfs_objects'
+    BATCH_SIZE = 10_000
+
+    def self.params_for_background_migration
+      yield self.where(file_store: nil), 'FillFileStoreLfsObject', 5.minutes, BATCH_SIZE
+    end
+  end
+
+  class Upload < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'uploads'
+    self.inheritance_column = :_type_disabled # Disable STI
+    BATCH_SIZE = 10_000
+
+    def self.params_for_background_migration
+      yield self.where(store: nil), 'FillStoreUpload', 5.minutes, BATCH_SIZE
+    end
+  end
+
+  def up
+    # NOTE: Schedule background migrations that fill 'NULL' value by '1'(ObjectStorage::Store::LOCAL) on `file_store`, `store` columns
+    # 
+    # Here are the target columns
+    # - ci_job_artifacts.file_store
+    # - lfs_objects.file_store
+    # - uploads.store
+
+    FillFileStore::JobArtifact.params_for_background_migration do |relation, class_name, delay_interval, batch_size|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+        class_name,
+        delay_interval,
+        batch_size: batch_size)
+    end
+
+    FillFileStore::LfsObject.params_for_background_migration do |relation, class_name, delay_interval, batch_size|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+        class_name,
+        delay_interval,
+        batch_size: batch_size)
+    end
+
+    FillFileStore::Upload.params_for_background_migration do |relation, class_name, delay_interval, batch_size|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+        class_name,
+        delay_interval,
+        batch_size: batch_size)
+    end
+  end
+
+  def down
+    # noop
+  end
+end

lib/gitlab/background_migration/fill_file_store_job_artifact.rb

+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreJobArtifact
+      class JobArtifact < ActiveRecord::Base
+        self.table_name = 'ci_job_artifacts'
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreJobArtifact::JobArtifact
+          .where(file_store: nil)
+          .where(id: (start_id..stop_id))
+          .update_all(file_store: 1)
+      end
+    end
+  end
+end

lib/gitlab/background_migration/fill_file_store_lfs_object.rb

+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillFileStoreLfsObject
+      class LfsObject < ActiveRecord::Base
+        self.table_name = 'lfs_objects'
+      end
+
+      def perform(start_id, stop_id)
+        FillFileStoreLfsObject::LfsObject
+          .where(file_store: nil)
+          .where(id: (start_id..stop_id))
+          .update_all(file_store: 1)
+      end
+    end
+  end
+end

lib/gitlab/background_migration/fill_store_upload.rb

+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class FillStoreUpload
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        FillStoreUpload::Upload
+          .where(store: nil)
+          .where(id: (start_id..stop_id))
+          .update_all(store: 1)
+      end
+    end
+  end
+end

spec/migrations/fill_file_store_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180424151928_fill_file_store')
+
+describe FillFileStore, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:builds) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+  let(:lfs_objects) { table(:lfs_objects) }
+  let(:uploads) { table(:uploads) }
+
+  before do
+    namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
+    builds.create!(id: 1)
+
+    ##
+    # Create rows that have nullfied `file_store` column
+    job_artifacts.create!(project_id: 123, job_id: 1, file_type: 1, file_store: nil)
+    lfs_objects.create!(oid: 123, size: 10, file: 'file_name', file_store: nil)
+    uploads.create!(size: 10, path: 'path', uploader: 'uploader', mount_point: 'file_name', store: nil)
+  end
+
+  it 'correctly migrates nullified file_store/store column' do
+    expect(job_artifacts.where(file_store: nil).count).to eq(1)
+    expect(lfs_objects.where(file_store: nil).count).to eq(1)
+    expect(uploads.where(store: nil).count).to eq(1)
+
+    expect(job_artifacts.where(file_store: 1).count).to eq(0)
+    expect(lfs_objects.where(file_store: 1).count).to eq(0)
+    expect(uploads.where(store: 1).count).to eq(0)
+
+    migrate!
+
+    expect(job_artifacts.where(file_store: nil).count).to eq(0)
+    expect(lfs_objects.where(file_store: nil).count).to eq(0)
+    expect(uploads.where(store: nil).count).to eq(0)
+
+    expect(job_artifacts.where(file_store: 1).count).to eq(1)
+    expect(lfs_objects.where(file_store: 1).count).to eq(1)
+    expect(uploads.where(store: 1).count).to eq(1)
+  end
+end