Merge branch 'add-ci-build-trace-metadata-table' into 'master'

Introduce a table to store job trace metadata

See merge request gitlab-org/gitlab!68171

app/models/ci/build.rb

@@ -53,6 +53,7 @@ module Ci
     end
 
     has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
+    has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build
 
     accepts_nested_attributes_for :runner_session, update_only: true
     accepts_nested_attributes_for :job_variables

app/models/ci/build_trace_metadata.rb

+# frozen_string_literal: true
+
+module Ci
+  class BuildTraceMetadata < Ci::ApplicationRecord
+    self.table_name = 'ci_build_trace_metadata'
+    self.primary_key = :build_id
+
+    belongs_to :build, class_name: 'Ci::Build'
+    belongs_to :trace_artifact, class_name: 'Ci::JobArtifact'
+
+    validates :build, presence: true
+  end
+end

db/migrate/20210813111909_create_ci_build_trace_metadata.rb

+# frozen_string_literal: true
+
+class CreateCiBuildTraceMetadata < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  def up
+    with_lock_retries do
+      create_table :ci_build_trace_metadata, id: false, if_not_exists: true do |t|
+        t.references :build,
+          index: false,
+          primary_key: true,
+          default: nil,
+          foreign_key: { to_table: :ci_builds, on_delete: :cascade },
+          type: :bigint,
+          null: false
+
+        t.bigint :trace_artifact_id
+        t.integer :archival_attempts, default: 0, null: false, limit: 2
+        t.binary :checksum
+        t.binary :remote_checksum
+
+        t.index :trace_artifact_id
+      end
+    end
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :ci_build_trace_metadata, if_exists: true
+    end
+  end
+end

db/migrate/20210817084338_add_foreign_key_from_ci_build_metadata_to_ci_job_artifacts.rb

+# frozen_string_literal: true
+
+class AddForeignKeyFromCiBuildMetadataToCiJobArtifacts < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :ci_build_trace_metadata,
+      :ci_job_artifacts,
+      column: :trace_artifact_id,
+      on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :ci_build_trace_metadata, column: :trace_artifact_id
+    end
+  end
+end

db/schema_migrations/20210813111909

+78ea79c2157acb8a0b29afb4d2f3de6965fb0ea885b5d5f939f22ccda1d53b1e
\ No newline at end of file

db/schema_migrations/20210817084338

+e301e1be82c96e62bba0f958c54dda4633b28553246d4c576d6de33cd33e0a50
\ No newline at end of file

db/structure.sql

@@ -10575,6 +10575,14 @@ CREATE SEQUENCE ci_build_trace_chunks_id_seq
 
 ALTER SEQUENCE ci_build_trace_chunks_id_seq OWNED BY ci_build_trace_chunks.id;
 
+CREATE TABLE ci_build_trace_metadata (
+    build_id bigint NOT NULL,
+    trace_artifact_id bigint,
+    archival_attempts smallint DEFAULT 0 NOT NULL,
+    checksum bytea,
+    remote_checksum bytea
+);
+
 CREATE TABLE ci_builds (
     id integer NOT NULL,
     status character varying,
@@ -21364,6 +21372,9 @@ ALTER TABLE ONLY ci_build_report_results
 ALTER TABLE ONLY ci_build_trace_chunks
     ADD CONSTRAINT ci_build_trace_chunks_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY ci_build_trace_metadata
+    ADD CONSTRAINT ci_build_trace_metadata_pkey PRIMARY KEY (build_id);
+
 ALTER TABLE ONLY dep_ci_build_trace_sections
     ADD CONSTRAINT ci_build_trace_sections_pkey PRIMARY KEY (build_id, section_name_id);
 
@@ -23322,6 +23333,8 @@ CREATE INDEX index_ci_build_report_results_on_project_id ON ci_build_report_resu
 
 CREATE UNIQUE INDEX index_ci_build_trace_chunks_on_build_id_and_chunk_index ON ci_build_trace_chunks USING btree (build_id, chunk_index);
 
+CREATE INDEX index_ci_build_trace_metadata_on_trace_artifact_id ON ci_build_trace_metadata USING btree (trace_artifact_id);
+
 CREATE UNIQUE INDEX index_ci_builds_metadata_on_build_id ON ci_builds_metadata USING btree (build_id);
 
 CREATE INDEX index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts ON ci_builds_metadata USING btree (build_id) WHERE (has_exposed_artifacts IS TRUE);
@@ -26200,6 +26213,9 @@ ALTER TABLE ONLY epics
 ALTER TABLE ONLY geo_container_repository_updated_events
     ADD CONSTRAINT fk_212c89c706 FOREIGN KEY (container_repository_id) REFERENCES container_repositories(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY ci_build_trace_metadata
+    ADD CONSTRAINT fk_21d25cac1a FOREIGN KEY (trace_artifact_id) REFERENCES ci_job_artifacts(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY users_star_projects
     ADD CONSTRAINT fk_22cd27ddfc FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 
@@ -28027,6 +28043,9 @@ ALTER TABLE ONLY analytics_cycle_analytics_group_stages
 ALTER TABLE ONLY metrics_dashboard_annotations
     ADD CONSTRAINT fk_rails_aeb11a7643 FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY ci_build_trace_metadata
+    ADD CONSTRAINT fk_rails_aebc78111f FOREIGN KEY (build_id) REFERENCES ci_builds(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY bulk_import_trackers
     ADD CONSTRAINT fk_rails_aed566d3f3 FOREIGN KEY (bulk_import_entity_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
 

spec/models/ci/build_spec.rb

@@ -28,6 +28,7 @@ RSpec.describe Ci::Build do
 
   it { is_expected.to have_one(:deployment) }
   it { is_expected.to have_one(:runner_session) }
+  it { is_expected.to have_one(:trace_metadata) }
 
   it { is_expected.to validate_presence_of(:ref) }
 

spec/models/ci/build_trace_metadata_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::BuildTraceMetadata do
+  it { is_expected.to belong_to(:build) }
+  it { is_expected.to belong_to(:trace_artifact) }
+
+  it { is_expected.to validate_presence_of(:build) }
+end

spec/services/ci/retry_build_service_spec.rb

@@ -60,7 +60,7 @@ RSpec.describe Ci::RetryBuildService do
        artifacts_file artifacts_metadata artifacts_size commands
        resource resource_group_id processed security_scans author
        pipeline_id report_results pending_state pages_deployments
-       queuing_entry runtime_metadata].freeze
+       queuing_entry runtime_metadata trace_metadata].freeze
 
   shared_examples 'build duplication' do
     let_it_be(:another_pipeline) { create(:ci_empty_pipeline, project: project) }