Merge branch '235729-add-pages_deployment-model' into 'master'

Add pages_deployment model

See merge request gitlab-org/gitlab!41785

app/models/ci/build.rb

@@ -47,6 +47,8 @@ module Ci
     has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id
     has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id
 
+    has_many :pages_deployments, inverse_of: :ci_build
+
     Ci::JobArtifact.file_types.each do |key, value|
       has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     end

app/models/ci/pipeline_artifact.rb

@@ -10,11 +10,6 @@ module Ci
     include FileStoreMounter
     include Presentable
 
-    FILE_STORE_SUPPORTED = [
-      ObjectStorage::Store::LOCAL,
-      ObjectStorage::Store::REMOTE
-    ].freeze
-
     FILE_SIZE_LIMIT = 10.megabytes.freeze
     EXPIRATION_DATE = 1.week.freeze
 
@@ -26,7 +21,7 @@ module Ci
     belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_artifacts
 
     validates :pipeline, :project, :file_format, :file, presence: true
-    validates :file_store, presence: true, inclusion: { in: FILE_STORE_SUPPORTED }
+    validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES }
     validates :size, presence: true, numericality: { less_than_or_equal_to: FILE_SIZE_LIMIT }
     validates :file_type, presence: true
 

app/models/pages_deployment.rb

+# frozen_string_literal: true
+
+# PagesDeployment stores a zip archive containing GitLab Pages web-site
+class PagesDeployment < ApplicationRecord
+  belongs_to :project, optional: false
+  belongs_to :ci_build, class_name: 'Ci::Build', optional: true
+
+  validates :file, presence: true
+  validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES }
+  validates :size, presence: true, numericality: { greater_than: 0, only_integer: true }
+end

app/models/project.rb

@@ -245,7 +245,6 @@ class Project < ApplicationRecord
   has_many :lfs_file_locks
   has_many :project_group_links
   has_many :invited_groups, through: :project_group_links, source: :group
-  has_many :pages_domains
   has_many :todos
   has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
 
@@ -327,8 +326,6 @@ class Project < ApplicationRecord
   has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_project_id
   has_many :source_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :project_id
 
-  has_one :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project
-
   has_many :import_failures, inverse_of: :project
   has_many :jira_imports, -> { order 'jira_imports.created_at' }, class_name: 'JiraImportState', inverse_of: :project
 
@@ -339,6 +336,11 @@ class Project < ApplicationRecord
   has_many :webide_pipelines, -> { webide_source }, class_name: 'Ci::Pipeline', inverse_of: :project
   has_many :reviews, inverse_of: :project
 
+  # GitLab Pages
+  has_many :pages_domains
+  has_one  :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project
+  has_many :pages_deployments
+
   # Can be too many records. We need to implement delete_all in batches.
   # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/228637
   has_many :product_analytics_events, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent

app/uploaders/object_storage.rb

@@ -30,6 +30,8 @@ module ObjectStorage
     REMOTE = 2
   end
 
+  SUPPORTED_STORES = [Store::LOCAL, Store::REMOTE].freeze
+
   module Extension
     # this extension is the glue between the ObjectStorage::Concern and RecordsUploads::Concern
     module RecordsUploads

changelogs/unreleased/235729-add-pages_deployment-model.yml

+---
+title: Add pages_deployments table
+merge_request: 41785
+author:
+type: added

db/migrate/20200911120132_create_pages_deployments.rb

+# frozen_string_literal: true
+
+class CreatePagesDeployments < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :pages_deployments, if_not_exists: true do |t|
+      t.timestamps_with_timezone
+
+      t.bigint :project_id, index: true, null: false
+      t.bigint :ci_build_id, index: true
+
+      t.integer :file_store, null: false, limit: 2
+      t.integer :size, null: false
+      t.text :file, null: false
+    end
+
+    add_text_limit :pages_deployments, :file, 255
+  end
+
+  def down
+    drop_table :pages_deployments
+  end
+end

db/migrate/20200911121027_add_pages_deployment_project_foreign_key.rb

+# frozen_string_literal: true
+
+class AddPagesDeploymentProjectForeignKey < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      add_foreign_key :pages_deployments, :projects, column: :project_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :pages_deployments, column: :project_id
+    end
+  end
+end

db/migrate/20200911121048_add_pages_deployment_ci_build_foreign_key.rb

+# frozen_string_literal: true
+
+class AddPagesDeploymentCiBuildForeignKey < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      add_foreign_key :pages_deployments, :ci_builds, column: :ci_build_id, on_delete: :nullify # rubocop:disable Migration/AddConcurrentForeignKey
+    end
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :pages_deployments, column: :ci_build_id
+    end
+  end
+end

db/schema_migrations/20200911120132

+8c9a7ead17d7378337e7bcfd25a10064c70a3bbb4604bdbb24065951131045dd
\ No newline at end of file

db/schema_migrations/20200911121027

+59cbcbe3f8491e150f37a3954c89cc58a52aa0a985c66f252c71d679429ffba1
\ No newline at end of file

db/schema_migrations/20200911121048

+51fe3e71baaf55730bf51f3ad5d429b7cfdc6c43319babb9af28bdc13aab2fc9
\ No newline at end of file

db/structure.sql

@@ -14179,6 +14179,27 @@ CREATE SEQUENCE public.packages_tags_id_seq
 
 ALTER SEQUENCE public.packages_tags_id_seq OWNED BY public.packages_tags.id;
 
+CREATE TABLE public.pages_deployments (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    project_id bigint NOT NULL,
+    ci_build_id bigint,
+    file_store smallint NOT NULL,
+    size integer NOT NULL,
+    file text NOT NULL,
+    CONSTRAINT check_f0fe8032dd CHECK ((char_length(file) <= 255))
+);
+
+CREATE SEQUENCE public.pages_deployments_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.pages_deployments_id_seq OWNED BY public.pages_deployments.id;
+
 CREATE TABLE public.pages_domain_acme_orders (
     id bigint NOT NULL,
     pages_domain_id integer NOT NULL,
@@ -17477,6 +17498,8 @@ ALTER TABLE ONLY public.packages_packages ALTER COLUMN id SET DEFAULT nextval('p
 
 ALTER TABLE ONLY public.packages_tags ALTER COLUMN id SET DEFAULT nextval('public.packages_tags_id_seq'::regclass);
 
+ALTER TABLE ONLY public.pages_deployments ALTER COLUMN id SET DEFAULT nextval('public.pages_deployments_id_seq'::regclass);
+
 ALTER TABLE ONLY public.pages_domain_acme_orders ALTER COLUMN id SET DEFAULT nextval('public.pages_domain_acme_orders_id_seq'::regclass);
 
 ALTER TABLE ONLY public.pages_domains ALTER COLUMN id SET DEFAULT nextval('public.pages_domains_id_seq'::regclass);
@@ -18681,6 +18704,9 @@ ALTER TABLE ONLY public.packages_pypi_metadata
 ALTER TABLE ONLY public.packages_tags
     ADD CONSTRAINT packages_tags_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY public.pages_deployments
+    ADD CONSTRAINT pages_deployments_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY public.pages_domain_acme_orders
     ADD CONSTRAINT pages_domain_acme_orders_pkey PRIMARY KEY (id);
 
@@ -20666,6 +20692,10 @@ CREATE INDEX index_packages_tags_on_package_id ON public.packages_tags USING btr
 
 CREATE INDEX index_packages_tags_on_package_id_and_updated_at ON public.packages_tags USING btree (package_id, updated_at DESC);
 
+CREATE INDEX index_pages_deployments_on_ci_build_id ON public.pages_deployments USING btree (ci_build_id);
+
+CREATE INDEX index_pages_deployments_on_project_id ON public.pages_deployments USING btree (project_id);
+
 CREATE INDEX index_pages_domain_acme_orders_on_challenge_token ON public.pages_domain_acme_orders USING btree (challenge_token);
 
 CREATE INDEX index_pages_domain_acme_orders_on_pages_domain_id ON public.pages_domain_acme_orders USING btree (pages_domain_id);
@@ -23184,6 +23214,9 @@ ALTER TABLE ONLY public.board_project_recent_visits
 ALTER TABLE ONLY public.clusters_kubernetes_namespaces
     ADD CONSTRAINT fk_rails_98fe21e486 FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY public.pages_deployments
+    ADD CONSTRAINT fk_rails_993b88f59a FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.vulnerability_exports
     ADD CONSTRAINT fk_rails_9aff2c3b45 FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE;
 
@@ -23391,6 +23424,9 @@ ALTER TABLE ONLY public.packages_nuget_dependency_link_metadata
 ALTER TABLE ONLY public.group_deploy_keys_groups
     ADD CONSTRAINT fk_rails_c3854f19f5 FOREIGN KEY (group_deploy_key_id) REFERENCES public.group_deploy_keys(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY public.pages_deployments
+    ADD CONSTRAINT fk_rails_c3a90cf29b FOREIGN KEY (ci_build_id) REFERENCES public.ci_builds(id) ON DELETE SET NULL;
+
 ALTER TABLE ONLY public.merge_request_user_mentions
     ADD CONSTRAINT fk_rails_c440b9ea31 FOREIGN KEY (note_id) REFERENCES public.notes(id) ON DELETE CASCADE;
 

spec/factories/ci/pipeline_artifacts.rb

@@ -6,7 +6,7 @@ FactoryBot.define do
     project { pipeline.project }
     file_type { :code_coverage }
     file_format { :raw }
-    file_store { Ci::PipelineArtifact::FILE_STORE_SUPPORTED.first }
+    file_store { ObjectStorage::SUPPORTED_STORES.first }
     size { 1.megabytes }
 
     after(:build) do |artifact, _evaluator|

spec/factories/pages_deployments.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :pages_deployment, class: 'PagesDeployment' do
+    project
+    file_store { ObjectStorage::SUPPORTED_STORES.first }
+    size { 1.megabytes }
+
+    # TODO: replace with proper file uploaded in https://gitlab.com/gitlab-org/gitlab/-/issues/245295
+    file { "dummy string" }
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -419,6 +419,8 @@ project:
 - project_feature
 - auto_devops
 - pages_domains
+- pages_metadatum
+- pages_deployments
 - authorized_users
 - project_authorizations
 - remote_mirrors
@@ -465,7 +467,6 @@ project:
 - approval_merge_request_rules
 - approvers
 - approver_users
-- pages_domains
 - audit_events
 - path_locks
 - approver_groups
@@ -505,7 +506,6 @@ project:
 - designs
 - project_aliases
 - external_pull_requests
-- pages_metadatum
 - alerts_service
 - grafana_integration
 - remove_source_branch_after_merge

spec/models/ci/build_spec.rb

@@ -25,6 +25,7 @@ RSpec.describe Ci::Build do
   it { is_expected.to have_many(:sourced_pipelines) }
   it { is_expected.to have_many(:job_variables) }
   it { is_expected.to have_many(:report_results) }
+  it { is_expected.to have_many(:pages_deployments) }
 
   it { is_expected.to have_one(:deployment) }
   it { is_expected.to have_one(:runner_session) }

spec/models/pages_deployment_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe PagesDeployment do
+  describe 'associations' do
+    it { is_expected.to belong_to(:project).required }
+    it { is_expected.to belong_to(:ci_build).optional }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:file) }
+    it { is_expected.to validate_presence_of(:size) }
+    it { is_expected.to validate_numericality_of(:size).only_integer.is_greater_than(0) }
+    it { is_expected.to validate_inclusion_of(:file_store).in_array(ObjectStorage::SUPPORTED_STORES) }
+
+    it 'is valid when created from the factory' do
+      expect(create(:pages_deployment)).to be_valid
+    end
+  end
+end

spec/models/project_spec.rb

@@ -85,7 +85,6 @@ RSpec.describe Project do
     it { is_expected.to have_many(:runners) }
     it { is_expected.to have_many(:variables) }
     it { is_expected.to have_many(:triggers) }
-    it { is_expected.to have_many(:pages_domains) }
     it { is_expected.to have_many(:labels).class_name('ProjectLabel') }
     it { is_expected.to have_many(:users_star_projects) }
     it { is_expected.to have_many(:repository_languages) }
@@ -125,6 +124,11 @@ RSpec.describe Project do
     it { is_expected.to have_many(:package_files).class_name('Packages::PackageFile') }
     it { is_expected.to have_many(:pipeline_artifacts) }
 
+    # GitLab Pages
+    it { is_expected.to have_many(:pages_domains) }
+    it { is_expected.to have_one(:pages_metadatum) }
+    it { is_expected.to have_many(:pages_deployments) }
+
     it_behaves_like 'model with repository' do
       let_it_be(:container) { create(:project, :repository, path: 'somewhere') }
       let(:stubbed_container) { build_stubbed(:project) }

spec/services/ci/retry_build_service_spec.rb

@@ -50,7 +50,7 @@ RSpec.describe Ci::RetryBuildService do
        metadata runner_session trace_chunks upstream_pipeline_id
        artifacts_file artifacts_metadata artifacts_size commands
        resource resource_group_id processed security_scans author
-       pipeline_id report_results pending_state].freeze
+       pipeline_id report_results pending_state pages_deployments].freeze
 
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }