Commit b63e87a8 authored by Krasimir Angelov's avatar Krasimir Angelov

Add support for namespaces to Pages internal API

Introduce new `project_pages_metadata` table, insert new record on
project creation. Update its `depoyed` flag when pages are
deployed/removed.

Return only these projects from namespace that have pages marked as
deployed. On-demand and mass data migration will handled in subsequent
commits.

Related to https://gitlab.com/gitlab-org/gitlab/issues/28781.
parent 64c60a33
......@@ -120,6 +120,13 @@ class Namespace < ApplicationRecord
uniquify = Uniquify.new
uniquify.string(path) { |s| Namespace.find_by_path_or_name(s) }
end
def find_by_pages_host(host)
gitlab_host = "." + Settings.pages.host.downcase
name = host.downcase.delete_suffix(gitlab_host)
Namespace.find_by_full_path(name)
end
end
def visibility_level_field
......@@ -305,8 +312,16 @@ class Namespace < ApplicationRecord
aggregation_schedule.present?
end
def pages_virtual_domain
Pages::VirtualDomain.new(all_projects_with_pages, trim_prefix: full_path)
end
private
def all_projects_with_pages
all_projects.with_pages_deployed
end
def parent_changed?
parent_id_changed?
end
......
......@@ -2,9 +2,10 @@
module Pages
class LookupPath
def initialize(project, domain: nil)
def initialize(project, trim_prefix: nil, domain: nil)
@project = project
@domain = domain
@trim_prefix = trim_prefix || project.full_path
end
def project_id
......@@ -28,11 +29,15 @@ module Pages
end
def prefix
'/'
if project.pages_group_root?
'/'
else
project.full_path.delete_prefix(trim_prefix) + '/'
end
end
private
attr_reader :project, :domain
attr_reader :project, :trim_prefix, :domain
end
end
......@@ -2,8 +2,9 @@
module Pages
class VirtualDomain
def initialize(projects, domain: nil)
def initialize(projects, trim_prefix: nil, domain: nil)
@projects = projects
@trim_prefix = trim_prefix
@domain = domain
end
......@@ -17,12 +18,12 @@ module Pages
def lookup_paths
projects.map do |project|
project.pages_lookup_path(domain: domain)
project.pages_lookup_path(trim_prefix: trim_prefix, domain: domain)
end.sort_by(&:prefix).reverse
end
private
attr_reader :projects, :domain
attr_reader :projects, :trim_prefix, :domain
end
end
......@@ -186,11 +186,17 @@ class PagesDomain < ApplicationRecord
end
def pages_virtual_domain
return unless pages_deployed?
Pages::VirtualDomain.new([project], domain: self)
end
private
def pages_deployed?
project.pages_metadatum&.deployed?
end
def set_verification_code
return if self.verification_code.present?
......
......@@ -104,6 +104,9 @@ class Project < ApplicationRecord
unless: :ci_cd_settings,
if: proc { ProjectCiCdSetting.available? }
after_create :create_pages_metadatum,
unless: :pages_metadatum
after_create :set_timestamps_for_create
after_update :update_forks_visibility_level
......@@ -295,6 +298,8 @@ class Project < ApplicationRecord
has_many :external_pull_requests, inverse_of: :project
has_one :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
......@@ -425,6 +430,10 @@ class Project < ApplicationRecord
.where(project_ci_cd_settings: { group_runners_enabled: true })
end
scope :with_pages_deployed, -> do
joins(:pages_metadatum).merge(ProjectPagesMetadatum.deployed)
end
enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
chronic_duration_attr :build_timeout_human_readable, :build_timeout,
......@@ -1643,6 +1652,10 @@ class Project < ApplicationRecord
"#{url}/#{url_path}"
end
def pages_group_root?
pages_group_url == pages_url
end
def pages_subdomain
full_path.partition('/').first
end
......@@ -1681,6 +1694,7 @@ class Project < ApplicationRecord
# Projects with a missing namespace cannot have their pages removed
return unless namespace
mark_pages_as_not_deployed unless destroyed?
::Projects::UpdatePagesConfigurationService.new(self).execute
# 1. We rename pages to temporary directory
......@@ -1694,6 +1708,14 @@ class Project < ApplicationRecord
end
# rubocop: enable CodeReuse/ServiceClass
def mark_pages_as_deployed
ensure_pages_metadatum.update!(deployed: true)
end
def mark_pages_as_not_deployed
ensure_pages_metadatum.update!(deployed: false)
end
# rubocop:disable Gitlab/RailsLogger
def write_repository_config(gl_full_path: full_path)
# We'd need to keep track of project full path otherwise directory tree
......@@ -2213,8 +2235,8 @@ class Project < ApplicationRecord
members.maintainers.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT)
end
def pages_lookup_path(domain: nil)
Pages::LookupPath.new(self, domain: domain)
def pages_lookup_path(trim_prefix: nil, domain: nil)
Pages::LookupPath.new(self, trim_prefix: trim_prefix, domain: domain)
end
private
......@@ -2342,6 +2364,13 @@ class Project < ApplicationRecord
def services_templates
@services_templates ||= Service.where(template: true)
end
def ensure_pages_metadatum
pages_metadatum || create_pages_metadatum!
rescue ActiveRecord::RecordNotUnique
reset
retry
end
end
Project.prepend_if_ee('EE::Project')
# frozen_string_literal: true
class ProjectPagesMetadatum < ApplicationRecord
self.primary_key = :project_id
belongs_to :project, inverse_of: :pages_metadatum
scope :deployed, -> { where(deployed: true) }
end
......@@ -53,6 +53,7 @@ module Projects
def success
@status.success
@project.mark_pages_as_deployed
super
end
......
---
title: Add project_pages_metadata DB table
merge_request: 17197
author:
type: added
# frozen_string_literal: true
class CreateProjectPagesMetadata < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :project_pages_metadata, id: false do |t|
t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
t.boolean :deployed, null: false, default: false
t.index :project_id, name: 'index_project_pages_metadata_on_project_id_and_deployed_is_true', where: "deployed = TRUE"
end
end
end
......@@ -2774,6 +2774,13 @@ ActiveRecord::Schema.define(version: 2019_09_19_162036) do
t.index ["status"], name: "index_project_mirror_data_on_status"
end
create_table "project_pages_metadata", id: false, force: :cascade do |t|
t.bigint "project_id", null: false
t.boolean "deployed", default: false, null: false
t.index ["project_id"], name: "index_project_pages_metadata_on_project_id", unique: true
t.index ["project_id"], name: "index_project_pages_metadata_on_project_id_and_deployed_is_true", where: "(deployed = true)"
end
create_table "project_repositories", force: :cascade do |t|
t.integer "shard_id", null: false
t.string "disk_path", null: false
......@@ -4084,6 +4091,7 @@ ActiveRecord::Schema.define(version: 2019_09_19_162036) do
add_foreign_key "project_incident_management_settings", "projects", on_delete: :cascade
add_foreign_key "project_metrics_settings", "projects", on_delete: :cascade
add_foreign_key "project_mirror_data", "projects", name: "fk_d1aad367d7", on_delete: :cascade
add_foreign_key "project_pages_metadata", "projects", on_delete: :cascade
add_foreign_key "project_repositories", "projects", on_delete: :cascade
add_foreign_key "project_repositories", "shards", on_delete: :restrict
add_foreign_key "project_repository_states", "projects", on_delete: :cascade
......
......@@ -17,11 +17,18 @@ module API
namespace 'internal' do
namespace 'pages' do
desc 'Get GitLab Pages domain configuration by hostname' do
detail 'This feature was introduced in GitLab 12.3.'
end
params do
requires :host, type: String, desc: 'The host to query for'
end
get "/" do
host = PagesDomain.find_by_domain(params[:host])
host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host])
not_found! unless host
virtual_domain = host.pages_virtual_domain
no_content! unless virtual_domain
present virtual_domain, with: Entities::Internal::Pages::VirtualDomain
end
......
......@@ -410,6 +410,7 @@ project:
- designs
- project_aliases
- external_pull_requests
- pages_metadatum
award_emoji:
- awardable
- user
......
......@@ -191,6 +191,16 @@ describe Namespace do
end
end
describe '.find_by_pages_host' do
it 'finds namespace by GitLab Pages host and is case-insensitive' do
namespace = create(:namespace, name: 'topnamespace')
create(:namespace, name: 'annother_namespace')
host = "TopNamespace.#{Settings.pages.host.upcase}"
expect(described_class.find_by_pages_host(host)).to eq(namespace)
end
end
describe '#ancestors_upto' do
let(:parent) { create(:group) }
let(:child) { create(:group, parent: parent) }
......@@ -913,4 +923,18 @@ describe Namespace do
end
end
end
describe '#pages_virtual_domain' do
let(:project) { create(:project, namespace: namespace) }
context 'when there are pages deployed for the project' do
before do
project.mark_pages_as_deployed
end
it 'returns the virual domain' do
expect(namespace.pages_virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
end
end
end
end
......@@ -57,8 +57,18 @@ describe Pages::LookupPath do
end
describe '#prefix' do
it 'returns "/"' do
it 'returns "/" for pages group root projects' do
project = instance_double(Project, pages_group_root?: true)
lookup_path = described_class.new(project, trim_prefix: 'mygroup')
expect(lookup_path.prefix).to eq('/')
end
it 'returns the project full path with the provided prefix removed' do
project = instance_double(Project, pages_group_root?: false, full_path: 'mygroup/myproject')
lookup_path = described_class.new(project, trim_prefix: 'mygroup')
expect(lookup_path.prefix).to eq('/myproject/')
end
end
end
......@@ -25,19 +25,33 @@ describe Pages::VirtualDomain do
end
describe '#lookup_paths' do
let(:domain) { instance_double(PagesDomain) }
let(:project_a) { instance_double(Project) }
let(:project_z) { instance_double(Project) }
let(:pages_lookup_path_a) { instance_double(Pages::LookupPath, prefix: 'aaa') }
let(:pages_lookup_path_z) { instance_double(Pages::LookupPath, prefix: 'zzz') }
subject(:virtual_domain) { described_class.new([project_a, project_z], domain: domain) }
context 'when there is pages domain provided' do
let(:domain) { instance_double(PagesDomain) }
it 'returns collection of projects pages lookup paths sorted by prefix in reverse' do
expect(project_a).to receive(:pages_lookup_path).with(domain: domain).and_return(pages_lookup_path_a)
expect(project_z).to receive(:pages_lookup_path).with(domain: domain).and_return(pages_lookup_path_z)
subject(:virtual_domain) { described_class.new([project_a, project_z], domain: domain) }
expect(virtual_domain.lookup_paths).to eq([pages_lookup_path_z, pages_lookup_path_a])
it 'returns collection of projects pages lookup paths sorted by prefix in reverse' do
expect(project_a).to receive(:pages_lookup_path).with(domain: domain, trim_prefix: nil).and_return(pages_lookup_path_a)
expect(project_z).to receive(:pages_lookup_path).with(domain: domain, trim_prefix: nil).and_return(pages_lookup_path_z)
expect(virtual_domain.lookup_paths).to eq([pages_lookup_path_z, pages_lookup_path_a])
end
end
context 'when there is trim_prefix provided' do
subject(:virtual_domain) { described_class.new([project_a, project_z], trim_prefix: 'group/') }
it 'returns collection of projects pages lookup paths sorted by prefix in reverse' do
expect(project_a).to receive(:pages_lookup_path).with(trim_prefix: 'group/', domain: nil).and_return(pages_lookup_path_a)
expect(project_z).to receive(:pages_lookup_path).with(trim_prefix: 'group/', domain: nil).and_return(pages_lookup_path_z)
expect(virtual_domain.lookup_paths).to eq([pages_lookup_path_z, pages_lookup_path_a])
end
end
end
end
......@@ -557,15 +557,27 @@ describe PagesDomain do
end
end
describe '.pages_virtual_domain' do
let(:project) { build(:project) }
describe '#pages_virtual_domain' do
let(:project) { create(:project) }
let(:pages_domain) { create(:pages_domain, project: project) }
subject(:pages_domain) { build(:pages_domain, project: project) }
context 'when there are no pages deployed for the project' do
it 'returns nil' do
expect(pages_domain.pages_virtual_domain).to be_nil
end
end
it 'returns instance of Pages::VirtualDomain' do
expect(Pages::VirtualDomain).to receive(:new).with([project], domain: pages_domain).and_call_original
context 'when there are pages deployed for the project' do
before do
project.mark_pages_as_deployed
project.reload
end
it 'returns the virual domain' do
expect(Pages::VirtualDomain).to receive(:new).with([project], domain: pages_domain).and_call_original
expect(pages_domain.pages_virtual_domain).to be_a(Pages::VirtualDomain)
expect(pages_domain.pages_virtual_domain).to be_an_instance_of(Pages::VirtualDomain)
end
end
end
end
......@@ -132,6 +132,13 @@ describe Project do
expect(project.ci_cd_settings).to be_an_instance_of(ProjectCiCdSetting)
expect(project.ci_cd_settings).to be_persisted
end
it 'automatically creates a Pages metadata row' do
project = create(:project)
expect(project.pages_metadatum).to be_an_instance_of(ProjectPagesMetadatum)
expect(project.pages_metadatum).to be_persisted
end
end
context 'updating cd_cd_settings' do
......@@ -3526,7 +3533,8 @@ describe Project do
end
describe '#remove_pages' do
let(:project) { create(:project) }
let(:project) { create(:project).tap { |project| project.mark_pages_as_deployed } }
let(:pages_metadatum) { project.pages_metadatum }
let(:namespace) { project.namespace }
let(:pages_path) { project.pages_path }
......@@ -3539,12 +3547,12 @@ describe Project do
end
end
it 'removes the pages directory' do
it 'removes the pages directory and marks the project as not having pages deployed' do
expect_any_instance_of(Projects::UpdatePagesConfigurationService).to receive(:execute)
expect_any_instance_of(Gitlab::PagesTransfer).to receive(:rename_project).and_return(true)
expect(PagesWorker).to receive(:perform_in).with(5.minutes, :remove, namespace.full_path, anything)
project.remove_pages
expect { project.remove_pages }.to change { pages_metadatum.reload.deployed }.from(true).to(false)
end
it 'is a no-op when there is no namespace' do
......@@ -3554,13 +3562,13 @@ describe Project do
expect_any_instance_of(Projects::UpdatePagesConfigurationService).not_to receive(:execute)
expect_any_instance_of(Gitlab::PagesTransfer).not_to receive(:rename_project)
project.remove_pages
expect { project.remove_pages }.not_to change { pages_metadatum.reload.deployed }
end
it 'is run when the project is destroyed' do
expect(project).to receive(:remove_pages).and_call_original
project.destroy
expect { project.destroy }.not_to raise_error
end
end
......@@ -5014,6 +5022,35 @@ describe Project do
end
end
context 'pages deployed' do
let(:project) { create(:project) }
{
mark_pages_as_deployed: true,
mark_pages_as_not_deployed: false
}.each do |method_name, flag|
describe method_name do
it "creates new record and sets deployed to #{flag} if none exists yet" do
project.pages_metadatum.destroy!
project.reload
project.send(method_name)
expect(project.pages_metadatum.reload.deployed).to eq(flag)
end
it "updates the existing record and sets deployed to #{flag}" do
pages_metadatum = project.pages_metadatum
pages_metadatum.update!(deployed: !flag)
expect { project.send(method_name) }.to change {
pages_metadatum.reload.deployed
}.from(!flag).to(flag)
end
end
end
end
describe '#has_pool_repsitory?' do
it 'returns false when it does not have a pool repository' do
subject = create(:project, :repository)
......@@ -5054,9 +5091,34 @@ describe Project do
let(:project) { build(:project) }
it 'returns instance of Pages::LookupPath' do
expect(Pages::LookupPath).to receive(:new).with(project, domain: pages_domain).and_call_original
expect(Pages::LookupPath).to receive(:new).with(project, domain: pages_domain, trim_prefix: 'mygroup').and_call_original
expect(project.pages_lookup_path(domain: pages_domain, trim_prefix: 'mygroup')).to be_a(Pages::LookupPath)
end
end
describe '.with_pages_deployed' do
it 'returns only projects that have pages deployed' do
_project_without_pages = create(:project)
project_with_pages = create(:project)
project_with_pages.mark_pages_as_deployed
expect(described_class.with_pages_deployed).to contain_exactly(project_with_pages)
end
end
describe '#pages_group_root?' do
it 'returns returns true if pages_url is same as pages_group_url' do
project = build(:project)
expect(project).to receive(:pages_url).and_return(project.pages_group_url)
expect(project.pages_group_root?).to eq(true)
end
it 'returns returns false if pages_url is different than pages_group_url' do
project = build(:project)
expect(project.pages_lookup_path(domain: pages_domain)).to be_a(Pages::LookupPath)
expect(project.pages_group_root?).to eq(false)
end
end
......
......@@ -43,6 +43,10 @@ describe API::Internal::Pages do
super(host, headers)
end
def deploy_pages(project)
project.mark_pages_as_deployed
end
context 'not existing host' do
it 'responds with 404 Not Found' do
query_host('pages.gitlab.io')
......@@ -56,18 +60,104 @@ describe API::Internal::Pages do
let(:project) { create(:project, namespace: namespace, name: 'gitlab-ce') }
let!(:pages_domain) { create(:pages_domain, domain: 'pages.gitlab.io', project: project) }
it 'responds with the correct domain configuration' do
query_host('pages.gitlab.io')
context 'when there are no pages deployed for the related project' do
it 'responds with 204 No Content' do
query_host('pages.gitlab.io')
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('internal/pages/virtual_domain')
expect(response).to have_gitlab_http_status(204)
end
end
expect(json_response['certificate']).to eq(pages_domain.certificate)
expect(json_response['key']).to eq(pages_domain.key)
context 'when there are pages deployed for the related project' do
it 'responds with the correct domain configuration' do
deploy_pages(project)
query_host('pages.gitlab.io')
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('internal/pages/virtual_domain')
expect(json_response['certificate']).to eq(pages_domain.certificate)
expect(json_response['key']).to eq(pages_domain.key)
expect(json_response['lookup_paths']).to eq(
[
{
'project_id' => project.id,
'access_control' => false,
'https_only' => false,
'prefix' => '/',
'source' => {
'type' => 'file',
'path' => 'gitlab-org/gitlab-ce/public/'
}
}
]
)
end
end
end
context 'namespaced domain' do
let(:group) { create(:group, name: 'mygroup') }
before do
allow(Settings.pages).to receive(:host).and_return('gitlab-pages.io')
allow(Gitlab.config.pages).to receive(:url).and_return("http://gitlab-pages.io")
end
context 'regular project' do
it 'responds with the correct domain configuration' do
project = create(:project, group: group, name: 'myproject')
deploy_pages(project)
query_host('mygroup.gitlab-pages.io')
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('internal/pages/virtual_domain')
expect(json_response['lookup_paths']).to eq(
[
{
'project_id' => project.id,
'access_control' => false,
'https_only' => false,
'prefix' => '/myproject/',
'source' => {
'type' => 'file',
'path' => 'mygroup/myproject/public/'
}
}
]
)
end
end
lookup_path = json_response['lookup_paths'][0]
expect(lookup_path['prefix']).to eq('/')
expect(lookup_path['source']['path']).to eq('gitlab-org/gitlab-ce/public/')
context 'group root project' do
it 'responds with the correct domain configuration' do
project = create(:project, group: group, name: 'mygroup.gitlab-pages.io')
deploy_pages(project)
query_host('mygroup.gitlab-pages.io')
expect(response).to have_gitlab_http_status(200)
expect(response).to match_response_schema('internal/pages/virtual_domain')
expect(json_response['lookup_paths']).to eq(
[
{
'project_id' => project.id,
'access_control' => false,
'https_only' => false,
'prefix' => '/',
'source' => {
'type' => 'file',
'path' => 'mygroup/mygroup.gitlab-pages.io/public/'
}
}
]
)
end
end
end
end
......
......@@ -40,6 +40,7 @@ describe Projects::UpdatePagesService do
it "doesn't delete artifacts after deploying" do
expect(execute).to eq(:success)
expect(project.pages_metadatum).to be_deployed
expect(build.artifacts?).to eq(true)
end
end
......@@ -47,6 +48,7 @@ describe Projects::UpdatePagesService do
it 'succeeds' do
expect(project.pages_deployed?).to be_falsey
expect(execute).to eq(:success)
expect(project.pages_metadatum).to be_deployed
expect(project.pages_deployed?).to be_truthy
# Check that all expected files are extracted
......@@ -63,16 +65,23 @@ describe Projects::UpdatePagesService do
it 'removes pages after destroy' do
expect(PagesWorker).to receive(:perform_in)
expect(project.pages_deployed?).to be_falsey
expect(execute).to eq(:success)
expect(project.pages_metadatum).to be_deployed
expect(project.pages_deployed?).to be_truthy
project.destroy
expect(project.pages_deployed?).to be_falsey
expect(ProjectPagesMetadatum.find_by_project_id(project)).to be_nil
end
it 'fails if sha on branch is not latest' do
build.update(ref: 'feature')
expect(execute).not_to eq(:success)
expect(project.pages_metadatum).not_to be_deployed
end
context 'when using empty file' do
......@@ -94,6 +103,7 @@ describe Projects::UpdatePagesService do
it 'succeeds to extract' do
expect(execute).to eq(:success)
expect(project.pages_metadatum).to be_deployed
end
end
end
......@@ -109,6 +119,7 @@ describe Projects::UpdatePagesService do
build.reload
expect(deploy_status).to be_failed
expect(project.pages_metadatum).not_to be_deployed
end
end
......@@ -125,6 +136,7 @@ describe Projects::UpdatePagesService do
build.reload
expect(deploy_status).to be_failed
expect(project.pages_metadatum).not_to be_deployed
end
end
......@@ -138,6 +150,7 @@ describe Projects::UpdatePagesService do
build.reload
expect(deploy_status).to be_failed
expect(project.pages_metadatum).not_to be_deployed
end
end
end
......@@ -179,6 +192,7 @@ describe Projects::UpdatePagesService do
expect(deploy_status.description)
.to match(/artifacts for pages are too large/)
expect(deploy_status).to be_script_failure
expect(project.pages_metadatum).not_to be_deployed
end
end
......@@ -196,6 +210,7 @@ describe Projects::UpdatePagesService do
subject.execute
expect(deploy_status.description).not_to be_present
expect(project.pages_metadatum).to be_deployed
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment