Merge branch 'mo-introduct-pipeline-artifact' into 'master'

Add PipelineArtifact table and model

See merge request gitlab-org/gitlab!37969

app/models/ci/pipeline.rb

@@ -83,6 +83,7 @@ module Ci
 
     has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id
     has_many :latest_builds_report_results, through: :latest_builds, source: :report_results
+    has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
     accepts_nested_attributes_for :variables, reject_if: :persisted?
 

app/models/ci/pipeline_artifact.rb

+# frozen_string_literal: true
+
+# This class is being used to persist generated report consumable by gitlab frontend in a pipeline context.
+
+module Ci
+  class PipelineArtifact < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    FILE_STORE_SUPPORTED = [
+      ObjectStorage::Store::LOCAL,
+      ObjectStorage::Store::REMOTE
+    ].freeze
+
+    FILE_SIZE_LIMIT = 10.megabytes.freeze
+
+    belongs_to :project, class_name: "Project", inverse_of: :pipeline_artifacts
+    belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_artifacts
+
+    validates :pipeline, :project, :file_format, presence: true
+    validates :file_store, presence: true, inclusion: { in: FILE_STORE_SUPPORTED }
+    validates :size, presence: true, numericality: { less_than_or_equal_to: FILE_SIZE_LIMIT }
+    validates :file_type, presence: true, uniqueness: { scope: [:pipeline_id] }
+
+    enum file_type: {
+      code_coverage: 1
+    }
+
+    enum file_format: {
+      raw: 1,
+      zip: 2,
+      gzip: 3
+    }, _suffix: true
+  end
+end

app/models/project.rb

@@ -299,6 +299,7 @@ class Project < ApplicationRecord
   has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks
   has_many :build_report_results, class_name: 'Ci::BuildReportResult', inverse_of: :project
   has_many :job_artifacts, class_name: 'Ci::JobArtifact'
+  has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :project
   has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'

changelogs/unreleased/mo-introduct-pipeline-artifact.yml

+---
+title: Add PipelineArtifact data model
+merge_request: 37969
+author:
+type: performance

db/migrate/20200805150316_create_ci_pipeline_artifact.rb

+# frozen_string_literal: true
+
+class CreateCiPipelineArtifact < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:ci_pipeline_artifacts)
+      create_table :ci_pipeline_artifacts do |t|
+        t.timestamps_with_timezone
+        t.bigint :pipeline_id, null: false, index: true
+        t.bigint :project_id, null: false, index: true
+        t.integer :size, null: false
+        t.integer :file_store, null: false, limit: 2
+        t.integer :file_type, null: false, limit: 2
+        t.integer :file_format, null: false, limit: 2
+        t.text :file
+
+        t.index [:pipeline_id, :file_type], unique: true
+      end
+    end
+
+    add_text_limit :ci_pipeline_artifacts, :file, 255
+  end
+
+  def down
+    drop_table :ci_pipeline_artifacts
+  end
+end

db/migrate/20200805151001_add_foreign_key_to_pipeline_id_on_pipeline_artifact.rb

+# frozen_string_literal: true
+
+class AddForeignKeyToPipelineIdOnPipelineArtifact < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      add_foreign_key :ci_pipeline_artifacts, :ci_pipelines, column: :pipeline_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :ci_pipeline_artifacts, column: :pipeline_id
+    end
+  end
+end

db/migrate/20200805151726_add_foreign_key_to_project_id_on_pipeline_artifact.rb

+# frozen_string_literal: true
+
+class AddForeignKeyToProjectIdOnPipelineArtifact < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      add_foreign_key :ci_pipeline_artifacts, :projects, column: :project_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :ci_pipeline_artifacts, column: :project_id
+    end
+  end
+end

db/schema_migrations/20200805150316

+983fa89c574e41a7d38eb558779f46d6d42aee5fba92f4185307e1508c401298
\ No newline at end of file

db/schema_migrations/20200805151001

+8fdb1e994ca7a28f7e061fb80cf210c482bafbe2bd0dc19c631c8fe9e0e2bbaf
\ No newline at end of file

db/schema_migrations/20200805151726

+e992135d6a4d10224b7e3deb304790735b6a35b5fb320670f9a7029e2924efb5
\ No newline at end of file

db/structure.sql

@@ -10047,6 +10047,29 @@ CREATE SEQUENCE public.ci_job_variables_id_seq
 
 ALTER SEQUENCE public.ci_job_variables_id_seq OWNED BY public.ci_job_variables.id;
 
+CREATE TABLE public.ci_pipeline_artifacts (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    pipeline_id bigint NOT NULL,
+    project_id bigint NOT NULL,
+    size integer NOT NULL,
+    file_store smallint NOT NULL,
+    file_type smallint NOT NULL,
+    file_format smallint NOT NULL,
+    file text,
+    CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255))
+);
+
+CREATE SEQUENCE public.ci_pipeline_artifacts_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.ci_pipeline_artifacts_id_seq OWNED BY public.ci_pipeline_artifacts.id;
+
 CREATE TABLE public.ci_pipeline_chat_data (
     id bigint NOT NULL,
     pipeline_id integer NOT NULL,
@@ -16710,6 +16733,8 @@ ALTER TABLE ONLY public.ci_job_artifacts ALTER COLUMN id SET DEFAULT nextval('pu
 
 ALTER TABLE ONLY public.ci_job_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_job_variables_id_seq'::regclass);
 
+ALTER TABLE ONLY public.ci_pipeline_artifacts ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_artifacts_id_seq'::regclass);
+
 ALTER TABLE ONLY public.ci_pipeline_chat_data ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_chat_data_id_seq'::regclass);
 
 ALTER TABLE ONLY public.ci_pipeline_messages ALTER COLUMN id SET DEFAULT nextval('public.ci_pipeline_messages_id_seq'::regclass);
@@ -17651,6 +17676,9 @@ ALTER TABLE ONLY public.ci_job_artifacts
 ALTER TABLE ONLY public.ci_job_variables
     ADD CONSTRAINT ci_job_variables_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY public.ci_pipeline_artifacts
+    ADD CONSTRAINT ci_pipeline_artifacts_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY public.ci_pipeline_chat_data
     ADD CONSTRAINT ci_pipeline_chat_data_pkey PRIMARY KEY (id);
 
@@ -19111,6 +19139,12 @@ CREATE INDEX index_ci_job_variables_on_job_id ON public.ci_job_variables USING b
 
 CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON public.ci_job_variables USING btree (key, job_id);
 
+CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON public.ci_pipeline_artifacts USING btree (pipeline_id);
+
+CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON public.ci_pipeline_artifacts USING btree (pipeline_id, file_type);
+
+CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON public.ci_pipeline_artifacts USING btree (project_id);
+
 CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON public.ci_pipeline_chat_data USING btree (chat_name_id);
 
 CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON public.ci_pipeline_chat_data USING btree (pipeline_id);
@@ -22142,6 +22176,9 @@ ALTER TABLE ONLY public.vulnerability_feedback
 ALTER TABLE ONLY public.user_custom_attributes
     ADD CONSTRAINT fk_rails_47b91868a8 FOREIGN KEY (user_id) REFERENCES public.users(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY public.ci_pipeline_artifacts
+    ADD CONSTRAINT fk_rails_4a70390ca6 FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.group_deletion_schedules
     ADD CONSTRAINT fk_rails_4b8c694a6c FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE;
 
@@ -22601,6 +22638,9 @@ ALTER TABLE ONLY public.resource_milestone_events
 ALTER TABLE ONLY public.term_agreements
     ADD CONSTRAINT fk_rails_a88721bcdf FOREIGN KEY (term_id) REFERENCES public.application_setting_terms(id);
 
+ALTER TABLE ONLY public.ci_pipeline_artifacts
+    ADD CONSTRAINT fk_rails_a9e811a466 FOREIGN KEY (pipeline_id) REFERENCES public.ci_pipelines(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.merge_request_user_mentions
     ADD CONSTRAINT fk_rails_aa1b2961b1 FOREIGN KEY (merge_request_id) REFERENCES public.merge_requests(id) ON DELETE CASCADE;
 

spec/factories/ci/pipeline_artifacts.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :ci_pipeline_artifact, class: 'Ci::PipelineArtifact' do
+    pipeline factory: :ci_pipeline
+    project { pipeline.project }
+    file_type { :code_coverage }
+    file_format { :raw }
+    file_store { Ci::PipelineArtifact::FILE_STORE_SUPPORTED.first }
+    size { 1.megabytes }
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -230,6 +230,7 @@ ci_pipelines:
 - daily_report_results
 - latest_builds_report_results
 - messages
+- pipeline_artifacts
 ci_refs:
 - project
 - ci_pipelines
@@ -519,6 +520,7 @@ project:
 - vulnerability_statistic
 - vulnerability_historical_statistics
 - product_analytics_events
+- pipeline_artifacts
 award_emoji:
 - awardable
 - user

spec/models/ci/pipeline_artifact_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::PipelineArtifact, type: :model do
+  let_it_be(:coverage_report) { create(:ci_pipeline_artifact) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:pipeline) }
+    it { is_expected.to belong_to(:project) }
+  end
+
+  it_behaves_like 'having unique enum values'
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:pipeline) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:file_type) }
+    it { is_expected.to validate_presence_of(:file_format) }
+    it { is_expected.to validate_presence_of(:size) }
+    it { is_expected.to validate_uniqueness_of(:file_type).scoped_to([:pipeline_id]).ignoring_case_sensitivity }
+
+    context 'when attributes are valid' do
+      it 'returns no errors' do
+        expect(coverage_report).to be_valid
+      end
+    end
+
+    context 'when file_store is invalid' do
+      it 'returns errors' do
+        coverage_report.file_store = 0
+
+        expect(coverage_report).to be_invalid
+        expect(coverage_report.errors.full_messages).to eq(["File store is not included in the list"])
+      end
+    end
+
+    context 'when size is over 10 megabytes' do
+      it 'returns errors' do
+        coverage_report.size = 11.megabytes
+
+        expect(coverage_report).to be_invalid
+      end
+    end
+  end
+end

spec/models/ci/pipeline_spec.rb

@@ -46,6 +46,7 @@ RSpec.describe Ci::Pipeline, :mailer do
   it { is_expected.to respond_to :git_author_email }
   it { is_expected.to respond_to :short_sha }
   it { is_expected.to delegate_method(:full_path).to(:project).with_prefix }
+  it { is_expected.to have_many(:pipeline_artifacts) }
 
   describe 'associations' do
     it 'has a bidirectional relationship with projects' do

spec/models/project_spec.rb

@@ -122,6 +122,7 @@ RSpec.describe Project do
     it { is_expected.to have_many(:reviews).inverse_of(:project) }
     it { is_expected.to have_many(:packages).class_name('Packages::Package') }
     it { is_expected.to have_many(:package_files).class_name('Packages::PackageFile') }
+    it { is_expected.to have_many(:pipeline_artifacts) }
 
     it_behaves_like 'model with repository' do
       let_it_be(:container) { create(:project, :repository, path: 'somewhere') }