Merge branch 'mo-add-pipeline-artifact-geo-database' into 'master'

Add geo database changes for pipeline artifact replication

See merge request gitlab-org/gitlab!57506

changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml

+---
+title: Add geo database changes for pipeline artifact replication
+merge_request: 57506
+author:
+type: added

db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb

+# frozen_string_literal: true
+
+class AddVerificationStateToCiPipelineArtifact < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    change_table(:ci_pipeline_artifacts, bulk: true) do |t|
+      t.column :verification_started_at, :datetime_with_timezone
+      t.column :verification_retry_at, :datetime_with_timezone
+      t.column :verified_at, :datetime_with_timezone
+      t.integer :verification_state, default: 0, limit: 2, null: false
+      t.integer :verification_retry_count, limit: 2
+      t.binary :verification_checksum, using: 'verification_checksum::bytea'
+
+      t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns
+    end
+  end
+end

db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb

+# frozen_string_literal: true
+
+class AddVerificationFailureLimitToCiPipelineArtifact < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  CONSTRAINT_NAME = 'ci_pipeline_artifacts_verification_failure_text_limit'
+
+  def up
+    add_text_limit :ci_pipeline_artifacts, :verification_failure, 255, constraint_name: CONSTRAINT_NAME
+  end
+
+  def down
+    remove_check_constraint(:ci_pipeline_artifacts, CONSTRAINT_NAME)
+  end
+end

db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb

+# frozen_string_literal: true
+
+class AddVerificationIndexesToCiPipelineArtifacts < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  VERIFICATION_STATE_INDEX_NAME = "index_ci_pipeline_artifacts_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_needs_verification"
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipeline_artifacts, :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+    add_concurrent_index :ci_pipeline_artifacts, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+    add_concurrent_index :ci_pipeline_artifacts, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+    add_concurrent_index :ci_pipeline_artifacts, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_pipeline_artifacts, VERIFICATION_STATE_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipeline_artifacts, PENDING_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipeline_artifacts, FAILED_VERIFICATION_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipeline_artifacts, NEEDS_VERIFICATION_INDEX_NAME
+  end
+end

db/schema_migrations/20210325150837

+6022464130d7a5697f52b9238837c6a6d3363fd349cbcb14052ff52de6ea2e59
\ No newline at end of file

db/schema_migrations/20210325151758

+de55a114773961e6cae9ebae36ac93e60676555fe4c2973527511bb3a2eae69d
\ No newline at end of file

db/schema_migrations/20210325152011

+379fdb3c52e55b51ebdb4a3b1e67c12f19b15e97cce22eed351e33953e389c85
\ No newline at end of file

db/structure.sql

@@ -10691,8 +10691,16 @@ CREATE TABLE ci_pipeline_artifacts (
     file_format smallint NOT NULL,
     file text,
     expire_at timestamp with time zone,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint,
+    verification_checksum bytea,
+    verification_failure text,
     CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)),
-    CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL))
+    CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)),
+    CONSTRAINT ci_pipeline_artifacts_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255))
 );
 
 CREATE SEQUENCE ci_pipeline_artifacts_id_seq
@@ -22204,6 +22212,10 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables
 
 CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date);
 
+CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
 CREATE INDEX index_ci_pipeline_artifacts_on_expire_at ON ci_pipeline_artifacts USING btree (expire_at);
 
 CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON ci_pipeline_artifacts USING btree (pipeline_id);
@@ -22212,6 +22224,10 @@ CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON
 
 CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON ci_pipeline_artifacts USING btree (project_id);
 
+CREATE INDEX index_ci_pipeline_artifacts_pending_verification ON ci_pipeline_artifacts USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
+CREATE INDEX index_ci_pipeline_artifacts_verification_state ON ci_pipeline_artifacts USING btree (verification_state);
+
 CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON ci_pipeline_chat_data USING btree (chat_name_id);
 
 CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON ci_pipeline_chat_data USING btree (pipeline_id);

ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb

+# frozen_string_literal: true
+
+class CreatePipelineArtifactRegistry < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :pipeline_artifact_registry, id: :bigserial, force: :cascade do |t|
+      t.bigint :pipeline_artifact_id, null: false
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :last_synced_at
+      t.datetime_with_timezone :retry_at
+      t.datetime_with_timezone :verified_at
+      t.datetime_with_timezone :verification_started_at
+      t.datetime_with_timezone :verification_retry_at
+      t.integer :state, default: 0, null: false, limit: 2
+      t.integer :verification_state, default: 0, null: false, limit: 2
+      t.integer :retry_count, default: 0, limit: 2
+      t.integer :verification_retry_count, default: 0, limit: 2
+      t.boolean :checksum_mismatch, default: false, null: false
+      t.binary :verification_checksum
+      t.binary :verification_checksum_mismatched
+      t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings
+      t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings
+
+      t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true
+      t.index :retry_at
+      t.index :state
+      t.index :verification_retry_at, name: :pipeline_artifact_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+      t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2)  AND (verification_state = ANY (ARRAY[0, 3])))"
+      t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+    end
+  end
+
+  def down
+    drop_table :pipeline_artifact_registry
+  end
+end

ee/db/geo/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_03_13_051642) do
+ActiveRecord::Schema.define(version: 2021_03_25_150435) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -160,7 +160,7 @@ ActiveRecord::Schema.define(version: 2021_03_13_051642) do
     t.integer "verification_state", limit: 2, default: 0, null: false
     t.integer "retry_count", limit: 2, default: 0
     t.integer "verification_retry_count", limit: 2, default: 0
-    t.boolean "checksum_mismatch"
+    t.boolean "checksum_mismatch", default: false, null: false
     t.binary "verification_checksum"
     t.binary "verification_checksum_mismatched"
     t.string "verification_failure", limit: 255