[ci skip] Merge branch 'master' into 43315-gpg-popover

* master: (39 commits)
  Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed
  Add changelog entry
  Use imported `DropdownUtils`
  Update CHANGELOG.md for 10.5.2
  Respond 404 when repo does not exist
  Fix deletion attempts on dropped tables
  Fix specs
  prevent localStorage.clear from running when it would cause an error
  Removed webpack tag
  move webpack bundles we intend to keep below the list of bundles we're refactoring to reduce conflicts
  Explicitly add uncovered files to troubleMakers
  Backport custom metrics ce components
  remove issue_show webpack bundle in favor of pages/projects/issues/show/index.js
  Revert "Revert "Use Dir.mktmpdir""
  Revert "Use Dir.mktmpdir"
  Use Dir.mktmpdir
  Refactored webpack bundle tag for deploy keys
  Change FileUtils.cp to FileUtils.copy
  Bring back the initial commit
  Add a button to deploy a runner to a Kubernetes cluster in the settings page
  ...

CHANGELOG.md

@@ -2,6 +2,27 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 10.5.2 (2018-02-25)
+
+### Fixed (7 changes)
+
+- Fix single digit value clipping for stacked progress bar. !17217
+- Fix issue with cache key being empty when variable used as the key. !17260
+- Enable Legacy Authorization by default on Cluster creations. !17302
+- Allow branch names to be named the same as the sha it points to.
+- Fix 500 error when loading an invalid upload URL.
+- Don't attempt to update user tracked fields if database is in read-only.
+- Prevent MR Widget error when no CI configured.
+
+### Performance (5 changes)
+
+- Improve query performance for snippets dashboard. !17088
+- Only check LFS integrity for first ref in a push to avoid timeout. !17098
+- Improve query performance of MembersFinder. !17190
+- Increase feature flag cache TTL to one hour.
+- Improve performance of searching for and autocompleting of users.
+
+
 ## 10.5.1 (2018-02-22)
 
 - No changes.

app/assets/javascripts/boards/boards_bundle.js → app/assets/javascripts/boards/index.js

@@ -24,7 +24,7 @@ import './components/new_list_dropdown';
 import './components/modal/index';
 import '../vue_shared/vue_resource_interceptor';
 
-$(() => {
+export default () => {
   const $boardApp = document.getElementById('board-app');
   const Store = gl.issueBoards.BoardsStore;
   const ModalStore = gl.issueBoards.ModalStore;
@@ -236,4 +236,4 @@ $(() => {
       </div>
     `,
   });
-});
+};

app/assets/javascripts/deploy_keys/index.js

 import Vue from 'vue';
 import deployKeysApp from './components/app.vue';
 
-document.addEventListener('DOMContentLoaded', () => new Vue({
+export default () => new Vue({
   el: document.getElementById('js-deploy-keys'),
   components: {
     deployKeysApp,
@@ -18,4 +18,4 @@ document.addEventListener('DOMContentLoaded', () => new Vue({
       },
     });
   },
-}));
+});

app/assets/javascripts/dispatcher.js

@@ -42,31 +42,34 @@ var Dispatcher;
         });
       });
 
-      switch (page) {
-        case 'projects:merge_requests:index':
-        case 'projects:issues:index':
-        case 'projects:issues:show':
-        case 'projects:issues:new':
-        case 'projects:issues:edit':
-        case 'projects:merge_requests:creations:new':
-        case 'projects:merge_requests:creations:diffs':
-        case 'projects:merge_requests:edit':
-        case 'projects:merge_requests:show':
-        case 'projects:commit:show':
-        case 'projects:activity':
-        case 'projects:commits:show':
-        case 'projects:show':
-        case 'groups:show':
-        case 'projects:tree:show':
-        case 'projects:find_file:show':
-        case 'projects:blob:show':
-        case 'projects:blame:show':
-        case 'projects:network:show':
-        case 'projects:artifacts:browse':
-        case 'projects:artifacts:file':
+      const shortcutHandlerPages = [
+        'projects:activity',
+        'projects:artifacts:browse',
+        'projects:artifacts:file',
+        'projects:blame:show',
+        'projects:blob:show',
+        'projects:commit:show',
+        'projects:commits:show',
+        'projects:find_file:show',
+        'projects:issues:edit',
+        'projects:issues:index',
+        'projects:issues:new',
+        'projects:issues:show',
+        'projects:merge_requests:creations:diffs',
+        'projects:merge_requests:creations:new',
+        'projects:merge_requests:edit',
+        'projects:merge_requests:index',
+        'projects:merge_requests:show',
+        'projects:network:show',
+        'projects:show',
+        'projects:tree:show',
+        'groups:show',
+      ];
+
+      if (shortcutHandlerPages.indexOf(page) !== -1) {
         shortcut_handler = true;
-          break;
       }
+
       switch (path[0]) {
         case 'admin':
           switch (path[1]) {

app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js → app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue

+<script>
 import eventHub from '../event_hub';
 import FilteredSearchTokenizer from '../filtered_search_tokenizer';
 
 export default {
   name: 'RecentSearchesDropdownContent',
-
   props: {
     items: {
       type: Array,
@@ -19,7 +19,6 @@ export default {
       required: true,
     },
   },
-
   computed: {
     processedItems() {
       return this.items.map((item) => {
@@ -42,7 +41,6 @@ export default {
       return this.items.length > 0;
     },
   },
-
   methods: {
     onItemActivated(text) {
       eventHub.$emit('recentSearchesItemSelected', text);
@@ -54,8 +52,9 @@ export default {
       eventHub.$emit('requestClearRecentSearches');
     },
   },
-
-  template: `
+};
+</script>
+<template>
   <div>
     <div
       v-if="!isLocalStorageAvailable"
@@ -65,16 +64,20 @@ export default {
     <ul v-else-if="hasItems">
       <li
         v-for="(item, index) in processedItems"
-          :key="index">
+        :key="`processed-items-${index}`"
+      >
         <button
           type="button"
           class="filtered-search-history-dropdown-item"
           @click="onItemActivated(item.text)">
           <span>
             <span
-                v-for="(token, tokenIndex) in item.tokens"
-                class="filtered-search-history-dropdown-token">
-                <span class="name">{{ token.prefix }}</span><span class="value">{{ token.suffix }}</span>
+              class="filtered-search-history-dropdown-token"
+              v-for="(token, index) in item.tokens"
+              :key="`dropdown-token-${index}`"
+            >
+              <span class="name">{{ token.prefix }}</span>
+              <span class="value">{{ token.suffix }}</span>
             </span>
           </span>
           <span class="filtered-search-history-dropdown-search-token">
@@ -98,5 +101,4 @@ export default {
       You don't have any recent searches
     </div>
   </div>
-  `,
-};
+</template>

app/assets/javascripts/filtered_search/recent_searches_root.js

 import Vue from 'vue';
-import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content';
+import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content.vue';
 import eventHub from './event_hub';
 
 class RecentSearchesRoot {
@@ -33,7 +33,7 @@ class RecentSearchesRoot {
     this.vm = new Vue({
       el: this.wrapperElement,
       components: {
-        'recent-searches-dropdown-content': RecentSearchesDropdownContent,
+        RecentSearchesDropdownContent,
       },
       data() { return state; },
       template: `

app/assets/javascripts/labels_select.js

@@ -213,7 +213,7 @@ export default class LabelsSelect {
             }
           }
           if (label.duplicate) {
-            color = gl.DropdownUtils.duplicateLabelColor(label.color);
+            color = DropdownUtils.duplicateLabelColor(label.color);
           }
           else {
             if (label.color != null) {

app/assets/javascripts/pages/projects/boards/index.js

 import UsersSelect from '~/users_select';
 import ShortcutsNavigation from '~/shortcuts_navigation';
+import initBoards from '~/boards';
 
 document.addEventListener('DOMContentLoaded', () => {
   new UsersSelect(); // eslint-disable-line no-new
   new ShortcutsNavigation(); // eslint-disable-line no-new
+  initBoards();
 });

app/assets/javascripts/pages/projects/issues/show/index.js

@@ -3,6 +3,7 @@ import Issue from '~/issue';
 import ShortcutsIssuable from '~/shortcuts_issuable';
 import ZenMode from '~/zen_mode';
 import '~/notes/index';
+import '~/issue_show/index';
 
 document.addEventListener('DOMContentLoaded', () => {
   new Issue(); // eslint-disable-line no-new

app/assets/javascripts/pages/projects/settings/repository/show/index.js

 import initSettingsPanels from '~/settings_panels';
+import initDeployKeys from '~/deploy_keys';
 
-document.addEventListener('DOMContentLoaded', initSettingsPanels);
+document.addEventListener('DOMContentLoaded', () => {
+  initDeployKeys();
+  initSettingsPanels();
+});

app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js

@@ -227,7 +227,8 @@ export default {
               @click="handleMergeButtonClick()"
               :disabled="isMergeButtonDisabled"
               :class="mergeButtonClass"
-              type="button">
+              type="button"
+              class="qa-merge-button">
               <i
                 v-if="isMakingRequest"
                 class="fa fa-spinner fa-spin"

app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue

@@ -111,7 +111,7 @@ js-toggle-container accept-action media space-children"
         >
           <button
             type="button"
-            class="btn btn-sm btn-reopen btn-success"
+            class="btn btn-sm btn-reopen btn-success qa-mr-rebase-button"
             :disabled="isMakingRequest"
             @click="rebase"
           >

app/controllers/projects/pages_domains_controller.rb

@@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController
 
   before_action :require_pages_enabled!
   before_action :authorize_update_pages!, except: [:show]
-  before_action :domain, only: [:show, :destroy]
+  before_action :domain, only: [:show, :destroy, :verify]
 
   def show
   end
@@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController
     @domain = @project.pages_domains.new
   end
 
+  def verify
+    result = VerifyPagesDomainService.new(@domain).execute
+
+    if result[:status] == :success
+      flash[:notice] = 'Successfully verified domain ownership'
+    else
+      flash[:alert] = 'Failed to verify domain ownership'
+    end
+
+    redirect_to project_pages_domain_path(@project, @domain)
+  end
+
   def create
     @domain = @project.pages_domains.create(pages_domain_params)
 
     if @domain.valid?
-      redirect_to project_pages_path(@project)
+      redirect_to project_pages_domain_path(@project, @domain)
     else
       render 'new'
     end
@@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController
   end
 
   def domain
-    @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s)
+    @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s)
   end
 end

app/controllers/projects/prometheus/metrics_controller.rb

+module Projects
+  module Prometheus
+    class MetricsController < Projects::ApplicationController
+      before_action :authorize_admin_project!
+
+      def active_common
+        respond_to do |format|
+          format.json do
+            matched_metrics = prometheus_service.matched_metrics || {}
+
+            if matched_metrics.any?
+              render json: matched_metrics
+            else
+              head :no_content
+            end
+          end
+        end
+      end
+
+      private
+
+      def prometheus_service
+        @prometheus_service ||= project.find_or_initialize_service('prometheus')
+      end
+    end
+  end
+end

app/controllers/projects/prometheus_controller.rb

-class Projects::PrometheusController < Projects::ApplicationController
-  before_action :authorize_read_project!
-  before_action :require_prometheus_metrics!
-
-  def active_metrics
-    respond_to do |format|
-      format.json do
-        matched_metrics = project.prometheus_service.matched_metrics || {}
-
-        if matched_metrics.any?
-          render json: matched_metrics
-        else
-          head :no_content
-        end
-      end
-    end
-  end
-
-  private
-
-  def require_prometheus_metrics!
-    render_404 unless project.prometheus_service.present?
-  end
-end

app/helpers/application_settings_helper.rb

@@ -199,6 +199,7 @@ module ApplicationSettingsHelper
       :metrics_port,
       :metrics_sample_interval,
       :metrics_timeout,
+      :pages_domain_verification_enabled,
       :password_authentication_enabled_for_web,
       :password_authentication_enabled_for_git,
       :performance_bar_allowed_group_id,

app/mailers/emails/pages_domains.rb

+module Emails
+  module PagesDomains
+    def pages_domain_enabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled")
+      )
+    end
+
+    def pages_domain_disabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled")
+      )
+    end
+
+    def pages_domain_verification_succeeded_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+
+    def pages_domain_verification_failed_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+  end
+end

app/mailers/notify.rb

@@ -5,6 +5,7 @@ class Notify < BaseMailer
   include Emails::Issues
   include Emails::MergeRequests
   include Emails::Notes
+  include Emails::PagesDomains
   include Emails::Projects
   include Emails::Profile
   include Emails::Pipelines

app/models/pages_domain.rb

 class PagesDomain < ActiveRecord::Base
+  VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze
+  VERIFICATION_THRESHOLD = 3.days.freeze
+
   belongs_to :project
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }
   validates :certificate, certificate: true, allow_nil: true, allow_blank: true
   validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+  validates :verification_code, presence: true, allow_blank: false
 
   validate :validate_pages_domain
   validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? }
@@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
+  after_initialize :set_verification_code
   after_create :update_daemon
-  after_save :update_daemon
+  after_update :update_daemon, if: :pages_config_changed?
   after_destroy :update_daemon
 
+  scope :enabled, -> { where('enabled_until >= ?', Time.now ) }
+  scope :needs_verification, -> do
+    verified_at = arel_table[:verified_at]
+    enabled_until = arel_table[:enabled_until]
+    threshold = Time.now + VERIFICATION_THRESHOLD
+
+    where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold))))
+  end
+
+  def verified?
+    !!verified_at
+  end
+
+  def unverified?
+    !verified?
+  end
+
+  def enabled?
+    !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present?
+  end
+
   def to_param
     domain
   end
@@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base
     @certificate_text ||= x509.try(:to_text)
   end
 
+  # Verification codes may be TXT records for domain or verification_domain, to
+  # support the use of CNAME records on domain.
+  def verification_domain
+    return unless domain.present?
+
+    "_#{VERIFICATION_KEY}.#{domain}"
+  end
+
+  def keyed_verification_code
+    return unless verification_code.present?
+
+    "#{VERIFICATION_KEY}=#{verification_code}"
+  end
+
   private
 
+  def set_verification_code
+    return if self.verification_code.present?
+
+    self.verification_code = SecureRandom.hex(16)
+  end
+
   def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
+  def pages_config_changed?
+    project_id_changed? ||
+      domain_changed? ||
+      certificate_changed? ||
+      key_changed? ||
+      became_enabled? ||
+      became_disabled?
+  end
+
+  def became_enabled?
+    enabled_until.present? && !enabled_until_was.present?
+  end
+
+  def became_disabled?
+    !enabled_until.present? && enabled_until_was.present?
+  end
+
   def validate_matching_key
     unless has_matching_key?
       self.errors.add(:key, "doesn't match the certificate")

app/models/project_services/prometheus_service.rb

@@ -69,16 +69,16 @@ class PrometheusService < MonitoringService
     client.ping
 
     { success: true, result: 'Checked API endpoint' }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err }
   end
 
   def environment_metrics(environment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
+    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics))
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics))
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics))
     metrics&.merge(deployment_time: deployment.created_at.to_i) || {}
   end
 
@@ -107,7 +107,7 @@ class PrometheusService < MonitoringService
       data: data,
       last_update: Time.now.utc
     }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err.message }
   end
 
@@ -116,10 +116,10 @@ class PrometheusService < MonitoringService
       Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url))
     else
       cluster = cluster_with_prometheus(environment_id)
-      raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster
+      raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster
 
       rest_client = client_from_cluster(cluster)
-      raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client
+      raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client
 
       Gitlab::PrometheusClient.new(rest_client)
     end
@@ -152,10 +152,12 @@ class PrometheusService < MonitoringService
     cluster.application_prometheus.proxy_client
   end
 
-  def rename_data_to_metrics(metrics)
-    metrics[:metrics] = metrics.delete :data
+  def rename_field(old_field, new_field)
+    -> (metrics) do
+      metrics[new_field] = metrics.delete(old_field)
       metrics
     end
+  end
 
   def synchronize_service_state!
     self.active = prometheus_installed? || manual_configuration?

app/services/ci/create_trace_artifact_service.rb

@@ -4,13 +4,33 @@ module Ci
       return if job.job_artifacts_trace
 
       job.trace.read do |stream|
-        if stream.file?
+        break unless stream.file?
+
+        clone_file!(stream.path, JobArtifactUploader.workhorse_upload_path) do |clone_path|
+          create_job_trace!(job, clone_path)
+          FileUtils.rm(stream.path)
+        end
+      end
+    end
+
+    private
+
+    def create_job_trace!(job, path)
+      File.open(path) do |stream|
         job.create_job_artifacts_trace!(
           project: job.project,
           file_type: :trace,
           file: stream)
       end
     end
+
+    def clone_file!(src_path, temp_dir)
+      FileUtils.mkdir_p(temp_dir)
+      Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path|
+        temp_path = File.join(dir_path, "job.log")
+        FileUtils.copy(src_path, temp_path)
+        yield(temp_path)
+      end
     end
   end
 end

app/services/notification_service.rb

@@ -339,6 +339,30 @@ class NotificationService
     end
   end
 
+  def pages_domain_verification_succeeded(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_verification_failed(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_failed_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_enabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_enabled_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_disabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_disabled_email(domain, user).deliver_later
+    end
+  end
+
   protected
 
   def new_resource_email(target, method)
@@ -433,6 +457,14 @@ class NotificationService
 
   private
 
+  def recipients_for_pages_domain(domain)
+    project = domain.project
+
+    return [] unless project
+
+    notifiable_users(project.team.masters, :watch, target: project)
+  end
+
   def notifiable?(*args)
     NotificationRecipientService.notifiable?(*args)
   end

app/services/projects/update_pages_configuration_service.rb

@@ -23,7 +23,7 @@ module Projects
     end
 
     def pages_domains_config
-      project.pages_domains.map do |domain|
+      enabled_pages_domains.map do |domain|
         {
           domain: domain.domain,
           certificate: domain.certificate,
@@ -32,6 +32,14 @@ module Projects
       end
     end
 
+    def enabled_pages_domains
+      if Gitlab::CurrentSettings.pages_domain_verification_enabled?
+        project.pages_domains.enabled
+      else
+        project.pages_domains
+      end
+    end
+
     def reload_daemon
       # GitLab Pages daemon constantly watches for modification time of `pages.path`
       # It reloads configuration when `pages.path` is modified

app/services/verify_pages_domain_service.rb

+require 'resolv'
+
+class VerifyPagesDomainService < BaseService
+  # The maximum number of seconds to be spent on each DNS lookup
+  RESOLVER_TIMEOUT_SECONDS = 15
+
+  # How long verification lasts for
+  VERIFICATION_PERIOD = 7.days
+
+  attr_reader :domain
+
+  def initialize(domain)
+    @domain = domain
+  end
+
+  def execute
+    return error("No verification code set for #{domain.domain}") unless domain.verification_code.present?
+
+    if !verification_enabled? || dns_record_present?
+      verify_domain!
+    elsif expired?
+      disable_domain!
+    else
+      unverify_domain!
+    end
+  end
+
+  private
+
+  def verify_domain!
+    was_disabled = !domain.enabled?
+    was_unverified = domain.unverified?
+
+    # Prevent any pre-existing grace period from being truncated
+    reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max
+
+    domain.update!(verified_at: Time.now, enabled_until: reverify)
+
+    if was_disabled
+      notify(:enabled)
+    elsif was_unverified
+      notify(:verification_succeeded)
+    end
+
+    success
+  end
+
+  def unverify_domain!
+    if domain.verified?
+      domain.update!(verified_at: nil)
+      notify(:verification_failed)
+    end
+
+    error("Couldn't verify #{domain.domain}")
+  end
+
+  def disable_domain!
+    domain.update!(verified_at: nil, enabled_until: nil)
+
+    notify(:disabled)
+
+    error("Couldn't verify #{domain.domain}. It is now disabled.")
+  end
+
+  # A domain is only expired until `disable!` has been called
+  def expired?
+    domain.enabled_until && domain.enabled_until < Time.now
+  end
+
+  def dns_record_present?
+    Resolv::DNS.open do |resolver|
+      resolver.timeouts = RESOLVER_TIMEOUT_SECONDS
+
+      check(domain.domain, resolver) || check(domain.verification_domain, resolver)
+    end
+  end
+
+  def check(domain_name, resolver)
+    records = parse(txt_records(domain_name, resolver))
+
+    records.any? do |record|
+      record == domain.keyed_verification_code || record == domain.verification_code
+    end
+  rescue => err
+    log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}")
+    false
+  end
+
+  def txt_records(domain_name, resolver)
+    resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT)
+  end
+
+  def parse(records)
+    records.flat_map(&:strings).flat_map(&:split)
+  end
+
+  def verification_enabled?
+    Gitlab::CurrentSettings.pages_domain_verification_enabled?
+  end
+
+  def notify(type)
+    return unless verification_enabled?
+
+    Gitlab::AppLogger.info("Pages domain '#{domain.domain}' changed state to '#{type}'")
+    notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -237,6 +237,17 @@
       .col-sm-10
         = f.number_field :max_pages_size, class: 'form-control'
         .help-block 0 for unlimited
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :pages_domain_verification_enabled do
+            = f.check_box :pages_domain_verification_enabled
+            Require users to prove ownership of custom domains
+        .help-block
+          Domain verification is an essential security measure for public GitLab
+          sites. Users are required to demonstrate they control a domain before
+          it is enabled
+          = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
 
   %fieldset
     %legend Continuous Integration and Deployment

app/views/admin/runners/index.html.haml

@@ -35,9 +35,8 @@
           method: :put, class: 'btn btn-default',
           data: { confirm: _("Are you sure you want to reset registration token?") }
 
-  = render partial: 'ci/runner/how_to_setup_runner',
-           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token,
-                     type: 'shared' }
+  = render partial: 'ci/runner/how_to_setup_shared_runner',
+           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token }
 
   .append-bottom-20.clearfix
     .pull-left

app/views/ci/runner/_how_to_setup_runner.html.haml

 - link = link_to _("GitLab Runner section"), 'https://about.gitlab.com/gitlab-ci/#gitlab-runner', target: '_blank'
-.bs-callout.help-callout
-  %h4= _("How to setup a #{type} Runner for a new project")
+.append-bottom-10
+  %h4= _("Setup a #{type} Runner manually")
 
-  %ol
+%ol
   %li
     = _("Install a Runner compatible with GitLab CI")
     = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe

app/views/ci/runner/_how_to_setup_shared_runner.html.haml

+.bs-callout.help-callout
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'shared' }

app/views/ci/runner/_how_to_setup_specific_runner.html.haml

+.bs-callout.help-callout
+  .append-bottom-10
+    %h4= _('Setup a specific Runner automatically')
+
+  %p
+    - link_to_help_page = link_to(_('Learn more about Kubernetes'),
+                                  help_page_path('user/project/clusters/index'),
+                                  target: '_blank',
+                                  rel: 'noopener noreferrer')
+
+    = _('You can easily install a Runner on a Kubernetes cluster. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page }
+
+  %ol
+    %li
+      = _('Click the button below to begin the install process by navigating to the Kubernetes page')
+    %li
+      = _('Select an existing Kubernetes cluster or create a new one')
+    %li
+      = _('From the Kubernetes cluster details view, install Runner from the applications list')
+
+  = link_to _('Install Runner on Kubernetes'),
+    project_clusters_path(@project),
+    class: 'btn btn-info'
+  %hr
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'specific' }

app/views/notify/pages_domain_disabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  %strong disabled.
+  This means that your content is no longer visible at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  If this domain has been disabled in error, please follow
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  to verify and re-enable your domain.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_disabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+**disabled**. This means that your content is no longer visible at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+If this domain has been disabled in error, please follow these instructions
+to verify and re-enable your domain:
+
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_enabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_enabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+enabled. You should now be able to view your content at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/notify/pages_domain_verification_failed_email.html.haml

+%p
+  Verification has failed for one of your GitLab Pages custom domains!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Unless you take action, it will be disabled on
+  %strong= @domain.enabled_until.strftime('%F %T.')
+  Until then, you can view your content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_failed_email.text.haml

+Verification has failed for one of your GitLab Pages custom domains!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*.
+Until then, you can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_succeeded_email.html.haml

+%p
+  One of your GitLab Pages custom domains has been successfully verified!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  This is a notification. No action is required on your part. You can view your
+  content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_verification_succeeded_email.text.haml

+One of your GitLab Pages custom domains has been successfully verified!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+No action is required on your part. You can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/projects/_merge_request_fast_forward_settings.html.haml

@@ -3,7 +3,7 @@
 
 .radio
   = label_tag :project_merge_method_ff do
-    = form.radio_button :merge_method, :ff, class: "js-merge-method-radio"
+    = form.radio_button :merge_method, :ff, class: "js-merge-method-radio qa-radio-button-merge-ff"
     %strong Fast-forward merge
     %br
     %span.descr

app/views/projects/clusters/_empty_state.html.haml

@@ -4,7 +4,7 @@
   .col-xs-12
     .text-content
       %h4.text-center= s_('ClusterIntegration|Integrate Kubernetes cluster automation')
-      - link_to_help_page = link_to(s_('ClusterIntegration|Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
+      - link_to_help_page = link_to(_('Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
       %p= s_('ClusterIntegration|Kubernetes clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page}
 
       .text-center

app/views/projects/edit.html.haml

@@ -85,7 +85,7 @@
     .settings-content
       = form_for [@project.namespace.becomes(Namespace), @project], remote: true, html: { multipart: true, class: "merge-request-settings-form" }, authenticity_token: true do |f|
         = render 'merge_request_settings', form: f
-        = f.submit 'Save changes', class: "btn btn-save"
+        = f.submit 'Save changes', class: "btn btn-save qa-save-merge-request-changes"
 
   = render 'export', project: @project
 

app/views/projects/issues/show.html.haml

@@ -82,6 +82,3 @@
     = render 'projects/issues/discussion'
 
 = render 'shared/issuable/sidebar', issuable: @issue
-
-= webpack_bundle_tag('common_vue')
-= webpack_bundle_tag('issue_show')

app/views/projects/pages/_list.html.haml

@@ -3,15 +3,26 @@
     .panel-heading
       Domains (#{@domains.count})
     %ul.well-list
+      - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
       - @domains.each do |domain|
         %li
           .pull-right
             = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped"
             = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped"
           .clearfix
-            %span= link_to domain.domain, domain.url
+            - if verification_enabled
+              - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success']
+              = link_to domain.url, title: tooltip, class: 'has-tooltip' do
+                = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}")
+                = domain.domain
+            - else
+              = link_to domain.domain, domain.url
           %p
             - if domain.subject
               %span.label.label-gray Certificate: #{domain.subject}
             - if domain.expired?
               %span.label.label-danger Expired
+        - if verification_enabled && domain.unverified?
+          %li.warning-row
+            #{domain.domain} is not verified. To learn how to verify ownership, visit your
+            = link_to 'domain details', project_pages_domain_path(@project, domain)

app/views/projects/pages_domains/show.html.haml

 - page_title "#{@domain.domain}", 'Pages Domains'
+- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
+- if verification_enabled && @domain.unverified?
+  %p.alert.alert-warning
+    %strong
+      This domain is not verified. You will need to verify ownership before
+      access is enabled.
 
 %h3.page-title
   Pages Domain
@@ -15,9 +21,26 @@
         DNS
       %td
         %p
-          To access the domain create a new DNS record:
+          To access this domain create a new DNS record:
         %pre
           #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}.
+    - if verification_enabled
+      %tr
+        %td
+          Verification status
+        %td
+          %p
+            - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+            To #{link_to 'verify ownership', help_link} of your domain, create
+            this DNS record:
+          %pre
+            #{@domain.verification_domain} TXT #{@domain.keyed_verification_code}
+          %p
+            - if @domain.verified?
+              #{@domain.domain} has been successfully verified.
+            - else
+              = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm'
+
     %tr
       %td
         Certificate

app/views/projects/runners/_shared_runners.html.haml

 %h3 Shared Runners
 
-.bs-callout.bs-callout-warning.shared-runners-description
+.bs-callout.shared-runners-description
   - if Gitlab::CurrentSettings.shared_runners_text.present?
     = markdown_field(Gitlab::CurrentSettings.current_application_settings, :shared_runners_text)
   - else
@@ -9,7 +9,7 @@
     on GitLab.com).
   %hr
   - if @project.shared_runners_enabled?
-    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do
+    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do
       Disable shared Runners
   - else
     = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do

app/views/projects/runners/_specific_runners.html.haml

 %h3 Specific Runners
 
-= render partial: 'ci/runner/how_to_setup_runner',
-         locals: { registration_token: @project.runners_token,
-                   type: 'specific' }
+= render partial: 'ci/runner/how_to_setup_specific_runner',
+         locals: { registration_token: @project.runners_token }
 
 - if @project_runners.any?
   %h4.underlined-title Runners activated for this project

app/views/projects/services/prometheus/_show.html.haml

@@ -7,7 +7,7 @@
       = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus')
 
   .col-lg-9
-    .panel.panel-default.js-panel-monitored-metrics{ data: { "active-metrics" => "#{project_prometheus_active_metrics_path(@project, :json)}" } }
+    .panel.panel-default.js-panel-monitored-metrics{ data: { active_metrics: active_common_project_prometheus_metrics_path(@project, :json) } }
       .panel-heading
         %h3.panel-title
           = s_('PrometheusService|Monitored')

app/views/projects/settings/repository/show.html.haml

@@ -4,7 +4,6 @@
 
 - content_for :page_specific_javascripts do
   = webpack_bundle_tag('common_vue')
-  = webpack_bundle_tag('deploy_keys')
 
 -# Protected branches & tags use a lot of nested partials.
 -# The shared parts of the views can be found in the `shared` directory.

app/views/shared/boards/_show.html.haml

@@ -6,7 +6,6 @@
 
 - content_for :page_specific_javascripts do
   = webpack_bundle_tag 'common_vue'
-  = webpack_bundle_tag 'boards'
 
   %script#js-board-template{ type: "text/x-template" }= render "shared/boards/components/board"
   %script#js-board-modal-filter{ type: "text/x-template" }= render "shared/issuable/search_bar", type: :boards_modal

app/workers/all_queues.yml

@@ -3,6 +3,7 @@
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup
+- cronjob:pages_domain_verification_cron
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
 - cronjob:remove_expired_group_links
@@ -82,6 +83,7 @@
 - new_merge_request
 - new_note
 - pages
+- pages_domain_verification
 - post_receive
 - process_commit
 - project_cache

app/workers/pages_domain_verification_cron_worker.rb

+class PagesDomainVerificationCronWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    PagesDomain.needs_verification.find_each do |domain|
+      PagesDomainVerificationWorker.perform_async(domain.id)
+    end
+  end
+end

app/workers/pages_domain_verification_worker.rb

+class PagesDomainVerificationWorker
+  include ApplicationWorker
+
+  def perform(domain_id)
+    domain = PagesDomain.find_by(id: domain_id)
+
+    return unless domain
+
+    VerifyPagesDomainService.new(domain).execute
+  end
+end

app/workers/stuck_import_jobs_worker.rb

@@ -16,43 +16,41 @@ class StuckImportJobsWorker
   private
 
   def mark_projects_without_jid_as_failed!
-    started_projects_without_jid.each do |project|
+    enqueued_projects_without_jid.each do |project|
       project.mark_import_as_failed(error_message)
     end.count
   end
 
   def mark_projects_with_jid_as_failed!
-    completed_jids_count = 0
-
-    started_projects_with_jid.find_in_batches(batch_size: 500) do |group|
-      jids = group.map(&:import_jid)
+    jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h
 
     # Find the jobs that aren't currently running or that exceeded the threshold.
-      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set
+    completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
+    return unless completed_jids.any?
 
-      if completed_jids.any?
-        completed_jids_count += completed_jids.count
-        group.each do |project|
-          project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid)
-        end
+    completed_project_ids = jids_and_ids.values_at(*completed_jids)
 
-        Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}")
-      end
-    end
+    # We select the projects again, because they may have transitioned from
+    # scheduled/started to finished/failed while we were looking up their Sidekiq status.
+    completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids)
 
-    completed_jids_count
+    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}")
+
+    completed_projects.each do |project|
+      project.mark_import_as_failed(error_message)
+    end.count
   end
 
-  def started_projects
-    Project.with_import_status(:started)
+  def enqueued_projects
+    Project.with_import_status(:scheduled, :started)
   end
 
-  def started_projects_with_jid
-    started_projects.where.not(import_jid: nil)
+  def enqueued_projects_with_jid
+    enqueued_projects.where.not(import_jid: nil)
   end
 
-  def started_projects_without_jid
-    started_projects.where(import_jid: nil)
+  def enqueued_projects_without_jid
+    enqueued_projects.where(import_jid: nil)
   end
 
   def error_message

changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml

+---
+title: Add verification for GitLab Pages custom domains
+merge_request:
+author:
+type: security

changelogs/unreleased/34130-null-pipes.yml

----
-title: Prevent MR Widget error when no CI configured
-merge_request:
-author:
-type: fixed

changelogs/unreleased/41461-project-members-slow-due-to-sql.yml

----
-title: Improve query performance of MembersFinder.
-merge_request: 17190
-author:
-type: performance

changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml

----
-title: Enable Legacy Authorization by default on Cluster creations
-merge_request: 17302
-author:
-type: fixed

changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml

+---
+title: Add a button to deploy a runner to a Kubernetes cluster in the settings page
+merge_request: 17278
+author:
+type: changed

changelogs/unreleased/42877-snippets-dashboard-slow.yml

----
-title: Improve query performance for snippets dashboard.
-merge_request: 17088
-author:
-type: performance

changelogs/unreleased/43373-fix-cache-index-appending.yml

----
-title: Fix issue with cache key being empty when variable used as the key
-merge_request: 17260
-author:
-type: fixed

changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml

+---
+title: Fix Group labels load failure when there are duplicate labels present
+merge_request: 17353
+author:
+type: fixed

changelogs/unreleased/fix-500-for-invalid-upload-path.yml → changelogs/unreleased/dm-stuck-import-jobs-verify.yml

 ---
-title: Fix 500 error when loading an invalid upload URL
+title: Verify project import status again before marking as failed
 merge_request:
 author:
 type: fixed

changelogs/unreleased/flipper-caching.yml

----
-title: Increase feature flag cache TTL to one hour
-merge_request:
-author:
-type: performance

changelogs/unreleased/grpc-unavailable-restart.yml

+---
+title: Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed
+merge_request: 17293
+author:
+type: fixed

changelogs/unreleased/jej-fix-slow-lfs-object-check.yml

----
-title: Only check LFS integrity for first ref in a push to avoid timeout
-merge_request: 17098
-author:
-type: performance

changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml

----
-title: Fix single digit value clipping for stacked progress bar
-merge_request: 17217
-author:
-type: fixed

changelogs/unreleased/minimal-fix-for-artifacts-service.yml

+---
+title: Prevent trace artifact migration to incur data loss
+merge_request: 17313
+author:
+type: fixed

changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml

+---
+title: Return a 404 instead of 403 if the repository does not exist on disk
+merge_request: 17341
+author:
+type: fixed

changelogs/unreleased/refactor-move-filtered-search-vue-component.yml

+---
+title: Move RecentSearchesDropdownContent vue component
+merge_request: 16951
+author: George Tsiolis
+type: performance

changelogs/unreleased/sh-guard-read-only-user-updates.yml

----
-title: Don't attempt to update user tracked fields if database is in read-only
-merge_request:
-author:
-type: fixed

changelogs/unreleased/users-autocomplete.yml

----
-title: Improve performance of searching for and autocompleting of users
-merge_request:
-author:
-type: performance

changelogs/unreleased/zj-branch-contains-git-message.yml

----
-title: Allow branch names to be named the same as the sha it points to
-merge_request:
-author:
-type: fixed

config/gitlab.yml.example

@@ -214,6 +214,10 @@ production: &base
     repository_archive_cache_worker:
       cron: "0 * * * *"
 
+    # Verify custom GitLab Pages domains
+    pages_domain_verification_cron_worker:
+      cron: "*/15 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
 
+Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *'
+Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker'
+
 #
 # GitLab Shell
 #

config/initializers/sidekiq.rb

@@ -10,7 +10,7 @@ Sidekiq.configure_server do |config|
 
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
-    chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
+    chain.add Gitlab::SidekiqMiddleware::Shutdown
     chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
     chain.add Gitlab::SidekiqStatus::ServerMiddleware
   end

config/routes/project.rb

@@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do
       end
 
       resource :pages, only: [:show, :destroy] do
-        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} }
+        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do
+          member do
+            post :verify
+          end
+        end
       end
 
       resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do
@@ -74,7 +78,9 @@ constraints(ProjectUrlConstrainer.new) do
       resource :mattermost, only: [:new, :create]
 
       namespace :prometheus do
-        get :active_metrics
+        resources :metrics, constraints: { id: %r{[^\/]+} }, only: [] do
+          get :active_common, on: :collection
+        end
       end
 
       resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do

config/sidekiq_queues.yml

@@ -67,3 +67,4 @@
   - [gcp_cluster, 1]
   - [project_migrate_hashed_storage, 1]
   - [storage_migrator, 1]
+  - [pages_domain_verification, 1]

config/webpack.config.js

@@ -52,20 +52,15 @@ var config = {
   entry: {
     balsamiq_viewer:      './blob/balsamiq_viewer.js',
     blob:                 './blob_edit/blob_bundle.js',
-    boards:               './boards/boards_bundle.js',
     common:               './commons/index.js',
     common_vue:           './vue_shared/vue_resource_interceptor.js',
     cycle_analytics:      './cycle_analytics/cycle_analytics_bundle.js',
     commit_pipelines:     './commit/pipelines/pipelines_bundle.js',
-    deploy_keys:          './deploy_keys/index.js',
     diff_notes:           './diff_notes/diff_notes_bundle.js',
     environments:         './environments/environments_bundle.js',
     environments_folder:  './environments/folder/environments_folder_bundle.js',
     filtered_search:      './filtered_search/filtered_search_bundle.js',
     help:                 './help/help.js',
-    issue_show:           './issue_show/index.js',
-    locale:               './locale/index.js',
-    main:                 './main.js',
     merge_conflicts:      './merge_conflicts/merge_conflicts_bundle.js',
     monitoring:           './monitoring/monitoring_bundle.js',
     network:              './network/network_bundle.js',
@@ -78,18 +73,24 @@ var config = {
     protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
     registry_list:        './registry/index.js',
-    ide:                  './ide/index.js',
     sidebar:              './sidebar/sidebar_bundle.js',
     snippet:              './snippet/snippet_bundle.js',
     sketch_viewer:        './blob/sketch_viewer.js',
     stl_viewer:           './blob/stl_viewer.js',
     terminal:             './terminal/terminal_bundle.js',
-    u2f:                  ['vendor/u2f'],
     ui_development_kit:   './ui_development_kit.js',
-    raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
-    test:                 './test.js',
     two_factor_auth:      './two_factor_auth.js',
+
+
+    common:               './commons/index.js',
+    common_vue:           './vue_shared/vue_resource_interceptor.js',
+    locale:               './locale/index.js',
+    main:                 './main.js',
+    ide:                  './ide/index.js',
+    raven:                './raven/index.js',
+    test:                 './test.js',
+    u2f:                  ['vendor/u2f'],
     webpack_runtime:      './webpack.js',
   },
 
@@ -252,7 +253,6 @@ var config = {
         'environments_folder',
         'filtered_search',
         'groups',
-        'issue_show',
         'merge_conflicts',
         'monitoring',
         'notebook_viewer',

db/migrate/20180216120000_add_pages_domain_verification.rb

+class AddPagesDomainVerification < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :verified_at, :datetime_with_timezone
+    add_column :pages_domains, :verification_code, :string
+  end
+end

db/migrate/20180216120010_add_pages_domain_verified_at_index.rb

+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, :verified_at
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, :verified_at
+  end
+end

db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb

+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+  end
+end

db/migrate/20180216120030_add_pages_domain_enabled_until.rb

+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :enabled_until, :datetime_with_timezone
+  end
+end

db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb

+class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, [:project_id, :enabled_until]
+    add_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+    remove_concurrent_index :pages_domains, [:project_id, :enabled_until]
+  end
+end

db/migrate/20180216120050_pages_domains_verification_grace_period.rb

+class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    now = Time.now
+    grace = now + 30.days
+
+    PagesDomain.each_batch do |relation|
+      relation.update_all(verified_at: now, enabled_until: grace)
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb

+class FillPagesDomainVerificationCode < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    PagesDomain.where(verification_code: [nil, '']).each_batch do |relation|
+      connection.execute(set_codes_sql(relation))
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+
+    change_column_null(:pages_domains, :verification_code, false)
+  end
+
+  def down
+    change_column_null(:pages_domains, :verification_code, true)
+  end
+
+  private
+
+  def set_codes_sql(relation)
+    ids = relation.pluck(:id)
+    whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" }
+
+    <<~SQL
+      UPDATE pages_domains
+      SET verification_code =
+        CASE id
+        #{whens.join("\n")}
+        END
+      WHERE id IN(#{ids.join(',')})
+    SQL
+  end
+end

db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb

+class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    PagesDomain.each_batch do |relation|
+      ids = relation.pluck(:id).map { |id| [id] }
+      PagesDomainVerificationWorker.bulk_perform_async(ids)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180215181245) do
+ActiveRecord::Schema.define(version: 20180216121030) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.integer "gitaly_timeout_fast", default: 10, null: false
     t.boolean "authorized_keys_enabled", default: true, null: false
     t.string "auto_devops_domain"
+    t.boolean "pages_domain_verification_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -1313,10 +1314,16 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.string "encrypted_key_iv"
     t.string "encrypted_key_salt"
     t.string "domain"
+    t.datetime_with_timezone "verified_at"
+    t.string "verification_code", null: false
+    t.datetime_with_timezone "enabled_until"
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
   add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
+  add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree
+  add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false

doc/administration/pages/index.md

@@ -226,6 +226,18 @@ world. Custom domains and TLS are supported.
 
 1. [Reconfigure GitLab][reconfigure]
 
+### Custom domain verification
+
+To prevent malicious users from hijacking domains that don't belong to them,
+GitLab supports [custom domain verification](../../user/project/pages/getting_started_part_three.md#dns-txt-record).
+When adding a custom domain, users will be required to prove they own it by
+adding a GitLab-controlled verification code to the DNS records for that domain.
+
+If your userbase is private or otherwise trusted, you can disable the
+verification requirement. Navigate to `Admin area ➔ Settings` and uncheck
+**Require users to prove ownership of custom domains** in the Pages section.
+This setting is enabled by default.
+
 ## Change storage path
 
 Follow the steps below to change the default path where GitLab Pages' contents

doc/user/project/pages/getting_started_part_three.md

@@ -62,7 +62,7 @@ for the most popular hosting services:
 - [Microsoft](https://msdn.microsoft.com/en-us/library/bb727018.aspx)
 
 If your hosting service is not listed above, you can just try to
-search the web for "how to add dns record on <my hosting service>".
+search the web for `how to add dns record on <my hosting service>`.
 
 ### DNS A record
 
@@ -95,12 +95,32 @@ without any `/project-name`.
 
 ![DNS CNAME record pointing to GitLab.com project](img/dns_cname_record_example.png)
 
-### TL;DR
+#### DNS TXT record
+
+Unless your GitLab administrator has [disabled custom domain verification](../../../administration/pages/index.md#custom-domain-verification),
+you'll have to prove that you own the domain by creating a `TXT` record
+containing a verification code. The code will be displayed after you
+[add your custom domain to GitLab Pages settings](#add-your-custom-domain-to-gitlab-pages-settings).
+
+If using a [DNS A record](#dns-a-record), you can place the TXT record directly
+under the domain. If using a [DNS CNAME record](#dns-cname-record), the two record types won't
+co-exist, so you need to place the TXT record in a special subdomain of its own.
+
+#### TL;DR
+
+If the domain has multiple uses (e.g., you host email on it as well):
 
 | From | DNS Record | To |
 | ---- | ---------- | -- |
 | domain.com | A | 52.167.214.135 |
-| subdomain.domain.com | CNAME | namespace.gitlab.io |
+| domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
+
+If the domain is dedicated to GitLab Pages use and no other services run on it:
+
+| From | DNS Record | To |
+| ---- | ---------- | -- |
+| subdomain.domain.com | CNAME | gitlab.io |
+| _gitlab-pages-verification-code.subdomain.domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
 
 > **Notes**:
 >
@@ -121,6 +141,17 @@ your site will be accessible only via HTTP:
 
 ![Add new domain](img/add_certificate_to_pages.png)
 
+Once you have added a new domain, you will need to **verify your ownership**
+(unless the GitLab administrator has disabled this feature). A verification code
+will be shown to you; add it as a [DNS TXT record](#dns-txt-record), then press
+the "Verify ownership" button to activate your new domain:
+
+![Verify your domain](img/verify_your_domain.png)
+
+Once your domain has been verified, leave the verification record in place -
+your domain will be periodically reverified, and may be disabled if the record
+is removed.
+
 You can add more than one alias (custom domains and subdomains) to the same project.
 An alias can be understood as having many doors leading to the same room.
 
@@ -128,8 +159,8 @@ All the aliases you've set to your site will be listed on **Setting > Pages**.
 From that page, you can view, add, and remove them.
 
 Note that [DNS propagation may take some time (up to 24h)](http://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/domain-names-dns-changes),
-although it's usually a matter of minutes to complete. Until it does, visit attempts
-to your domain will respond with a 404.
+although it's usually a matter of minutes to complete. Until it does, verification
+will fail and attempts to visit your domain will respond with a 404.
 
 Read through the [general documentation on GitLab Pages](introduction.md#add-a-custom-domain-to-your-pages-website) to learn more about adding
 custom domains to GitLab Pages sites.

lib/api/entities.rb

@@ -1154,6 +1154,10 @@ module API
       expose :domain
       expose :url
       expose :project_id
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         as: :certificate_expiration,
         if: ->(pages_domain, _) { pages_domain.certificate? },
@@ -1165,6 +1169,10 @@ module API
     class PagesDomain < Grape::Entity
       expose :domain
       expose :url
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         if: ->(pages_domain, _) { pages_domain.certificate? },
         using: PagesDomainCertificate do |pages_domain|

lib/gitlab/git_access.rb

@@ -199,7 +199,7 @@ module Gitlab
 
     def check_repository_existence!
       unless repository.exists?
-        raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
+        raise NotFoundError, ERROR_MESSAGES[:no_repo]
       end
     end
 

lib/gitlab/gitaly_client.rb

@@ -125,6 +125,8 @@ module Gitlab
       kwargs = yield(kwargs) if block_given?
 
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
+    rescue GRPC::Unavailable => ex
+      handle_grpc_unavailable!(ex)
     ensure
       duration = Gitlab::Metrics::System.monotonic_time - start
 
@@ -135,6 +137,27 @@ module Gitlab
         duration)
     end
 
+    def self.handle_grpc_unavailable!(ex)
+      status = ex.to_status
+      raise ex unless status.details == 'Endpoint read failed'
+
+      # There is a bug in grpc 1.8.x that causes a client process to get stuck
+      # always raising '14:Endpoint read failed'. The only thing that we can
+      # do to recover is to restart the process.
+      #
+      # See https://gitlab.com/gitlab-org/gitaly/issues/1029
+
+      if Sidekiq.server?
+        raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s)
+      else
+        # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request.
+        Process.kill('QUIT', Process.pid)
+      end
+
+      raise ex
+    end
+    private_class_method :handle_grpc_unavailable!
+
     def self.current_transaction_labels
       Gitlab::Metrics::Transaction.current&.labels || {}
     end

lib/gitlab/prometheus/metric_group.rb

@@ -6,9 +6,14 @@ module Gitlab
       attr_accessor :name, :priority, :metrics
       validates :name, :priority, :metrics, presence: true
 
-      def self.all
+      def self.common_metrics
         AdditionalMetricsParser.load_groups_from_yaml
       end
+
+      # EE only
+      def self.for_project(_)
+        common_metrics
+      end
     end
   end
 end

lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
+              deployment.project,
               common_query_context(
                 deployment.environment,
                 timeframe_start: (deployment.created_at - 30.minutes).to_f,

lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id)
           ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
+              environment.project,
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
           end

lib/gitlab/prometheus/queries/matched_metrics_query.rb

@@ -18,7 +18,7 @@ module Gitlab
         private
 
         def groups_data
-          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all)
+          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics)
           lookup = active_series_lookup(metrics_groups)
 
           groups = {}

lib/gitlab/prometheus/queries/query_additional_metrics.rb

@@ -2,10 +2,10 @@ module Gitlab
   module Prometheus
     module Queries
       module QueryAdditionalMetrics
-        def query_metrics(query_context)
+        def query_metrics(project, query_context)
           query_processor = method(:process_query).curry[query_context]
 
-          groups = matched_metrics.map do |group|
+          groups = matched_metrics(project).map do |group|
             metrics = group.metrics.map do |metric|
               {
                 title: metric.title,
@@ -60,8 +60,8 @@ module Gitlab
           @available_metrics ||= client_label_values || []
         end
 
-        def matched_metrics
-          result = Gitlab::Prometheus::MetricGroup.all.map do |group|
+        def matched_metrics(project)
+          result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group|
             group.metrics.select! do |metric|
               metric.required_metrics.all?(&available_metrics.method(:include?))
             end

lib/gitlab/prometheus_client.rb

 module Gitlab
-  PrometheusError = Class.new(StandardError)
-
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
+    Error = Class.new(StandardError)
+    QueryError = Class.new(Gitlab::PrometheusClient::Error)
+
     attr_reader :rest_client, :headers
 
     def initialize(rest_client)
@@ -43,22 +44,22 @@ module Gitlab
       path = ['api', 'v1', type].join('/')
       get(path, args)
     rescue JSON::ParserError
-      raise PrometheusError, 'Parsing response failed'
+      raise PrometheusClient::Error, 'Parsing response failed'
     rescue Errno::ECONNREFUSED
-      raise PrometheusError, 'Connection refused'
+      raise PrometheusClient::Error, 'Connection refused'
     end
 
     def get(path, args)
       response = rest_client[path].get(params: args)
       handle_response(response)
     rescue SocketError
-      raise PrometheusError, "Can't connect to #{rest_client.url}"
+      raise PrometheusClient::Error, "Can't connect to #{rest_client.url}"
     rescue OpenSSL::SSL::SSLError
-      raise PrometheusError, "#{rest_client.url} contains invalid SSL data"
+      raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data"
     rescue RestClient::ExceptionWithResponse => ex
       handle_exception_response(ex.response)
     rescue RestClient::Exception
-      raise PrometheusError, "Network connection error"
+      raise PrometheusClient::Error, "Network connection error"
     end
 
     def handle_response(response)
@@ -66,16 +67,18 @@ module Gitlab
       if response.code == 200 && json_data['status'] == 'success'
         json_data['data'] || {}
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 
     def handle_exception_response(response)
-      if response.code == 400
+      if response.code == 200 && response['status'] == 'success'
+        response['data'] || {}
+      elsif response.code == 400
         json_data = JSON.parse(response.body)
-        raise PrometheusError, json_data['error'] || 'Bad data received'
+        raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received'
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 

lib/gitlab/sidekiq_middleware/memory_killer.rb

-module Gitlab
-  module SidekiqMiddleware
-    class MemoryKiller
-      # Default the RSS limit to 0, meaning the MemoryKiller is disabled
-      MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
-      # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
-      GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
-      # Wait 30 seconds for running jobs to finish during graceful shutdown
-      SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
-
-      # Create a mutex used to ensure there will be only one thread waiting to
-      # shut Sidekiq down
-      MUTEX = Mutex.new
-
-      def call(worker, job, queue)
-        yield
-
-        current_rss = get_rss
-
-        return unless MAX_RSS > 0 && current_rss > MAX_RSS
-
-        Thread.new do
-          # Return if another thread is already waiting to shut Sidekiq down
-          return unless MUTEX.try_lock
-
-          Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\
-            " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}"
-          Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
-
-          # Wait `GRACE_TIME` to give the memory intensive job time to finish.
-          # Then, tell Sidekiq to stop fetching new jobs.
-          wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs')
-
-          # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
-          # Then, tell Sidekiq to gracefully shut down by giving jobs a few more
-          # moments to finish, killing and requeuing them if they didn't, and
-          # then terminating itself.
-          wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down')
-
-          # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
-          wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die')
-        end
-      end
-
-      private
-
-      def get_rss
-        output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
-        return 0 unless status.zero?
-
-        output.to_i
-      end
-
-      def wait_and_signal(time, signal, explanation)
-        Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
-        sleep(time)
-
-        Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
-        Process.kill(signal, pid)
-      end
-
-      def pid
-        Process.pid
-      end
-    end
-  end
-end

lib/gitlab/sidekiq_middleware/shutdown.rb

+require 'mutex_m'
+
+module Gitlab
+  module SidekiqMiddleware
+    class Shutdown
+      extend Mutex_m
+
+      # Default the RSS limit to 0, meaning the MemoryKiller is disabled
+      MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
+      # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
+      GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+      # Wait 30 seconds for running jobs to finish during graceful shutdown
+      SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+
+      # This exception can be used to request that the middleware start shutting down Sidekiq
+      WantShutdown = Class.new(StandardError)
+
+      ShutdownWithoutRaise = Class.new(WantShutdown)
+      private_constant :ShutdownWithoutRaise
+
+      # For testing only, to avoid race conditions (?) in Rspec mocks.
+      attr_reader :trace
+
+      # We store the shutdown thread in a class variable to ensure that there
+      # can be only one shutdown thread in the process.
+      def self.create_shutdown_thread
+        mu_synchronize do
+          return unless @shutdown_thread.nil?
+
+          @shutdown_thread = Thread.new { yield }
+        end
+      end
+
+      # For testing only: so we can wait for the shutdown thread to finish.
+      def self.shutdown_thread
+        mu_synchronize { @shutdown_thread }
+      end
+
+      # For testing only: so that we can reset the global state before each test.
+      def self.clear_shutdown_thread
+        mu_synchronize { @shutdown_thread = nil }
+      end
+
+      def initialize
+        @trace = Queue.new if Rails.env.test?
+      end
+
+      def call(worker, job, queue)
+        shutdown_exception = nil
+
+        begin
+          yield
+          check_rss!
+        rescue WantShutdown => ex
+          shutdown_exception = ex
+        end
+
+        return unless shutdown_exception
+
+        self.class.create_shutdown_thread do
+          do_shutdown(worker, job, shutdown_exception)
+        end
+
+        raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise)
+      end
+
+      private
+
+      def do_shutdown(worker, job, shutdown_exception)
+        Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\
+          "#{worker.class} JID-#{job['jid']}"
+        Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
+
+        # Wait `GRACE_TIME` to give the memory intensive job time to finish.
+        # Then, tell Sidekiq to stop fetching new jobs.
+        wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs')
+
+        # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
+        # Then, tell Sidekiq to gracefully shut down by giving jobs a few more
+        # moments to finish, killing and requeuing them if they didn't, and
+        # then terminating itself.
+        wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down')
+
+        # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
+        wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die')
+      end
+
+      def check_rss!
+        return unless MAX_RSS > 0
+
+        current_rss = get_rss
+        return unless current_rss > MAX_RSS
+
+        raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}")
+      end
+
+      def get_rss
+        output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
+        return 0 unless status.zero?
+
+        output.to_i
+      end
+
+      def wait_and_signal(time, signal, explanation)
+        Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
+        sleep(time)
+
+        Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
+        kill(signal, pid)
+      end
+
+      def pid
+        Process.pid
+      end
+
+      def sleep(time)
+        if Rails.env.test?
+          @trace << [:sleep, time]
+        else
+          Kernel.sleep(time)
+        end
+      end
+
+      def kill(signal, pid)
+        if Rails.env.test?
+          @trace << [:kill, signal, pid]
+        else
+          Process.kill(signal, pid)
+        end
+      end
+    end
+  end
+end

qa/qa.rb

@@ -130,6 +130,7 @@ module QA
         autoload :DeployKeys, 'qa/page/project/settings/deploy_keys'
         autoload :SecretVariables, 'qa/page/project/settings/secret_variables'
         autoload :Runners, 'qa/page/project/settings/runners'
+        autoload :MergeRequest, 'qa/page/project/settings/merge_request'
       end
 
       module Issue
@@ -145,6 +146,7 @@ module QA
 
     module MergeRequest
       autoload :New, 'qa/page/merge_request/new'
+      autoload :Show, 'qa/page/merge_request/show'
     end
 
     module Admin

qa/qa/factory/base.rb

@@ -22,7 +22,7 @@ module QA
 
           factory.fabricate!(*args)
 
-          return Factory::Product.populate!(self)
+          return Factory::Product.populate!(factory)
         end
       end
 

qa/qa/factory/product.rb

@@ -17,8 +17,9 @@ module QA
 
       def self.populate!(factory)
         new.tap do |product|
-          factory.attributes.each_value do |attribute|
-            product.instance_exec(&attribute.block).tap do |value|
+          factory.class.attributes.each_value do |attribute|
+            product.instance_exec(factory, attribute.block) do |factory, block|
+              value = block.call(factory)
               product.define_singleton_method(attribute.name) { value }
             end
           end

qa/qa/factory/repository/push.rb

@@ -2,7 +2,7 @@ module QA
   module Factory
     module Repository
       class Push < Factory::Base
-        attr_writer :file_name, :file_content, :commit_message, :branch_name
+        attr_writer :file_name, :file_content, :commit_message, :branch_name, :new_branch
 
         dependency Factory::Resource::Project, as: :project do |project|
           project.name = 'project-with-code'
@@ -14,6 +14,7 @@ module QA
           @file_content = '# This is test project'
           @commit_message = "Add #{@file_name}"
           @branch_name = 'master'
+          @new_branch = true
         end
 
         def fabricate!
@@ -29,6 +30,7 @@ module QA
             repository.clone
             repository.configure_identity('GitLab QA', 'root@gitlab.com')
 
+            repository.checkout(@branch_name) unless @new_branch
             repository.add_file(@file_name, @file_content)
             repository.commit(@commit_message)
             repository.push_changes(@branch_name)

qa/qa/factory/resource/merge_request.rb

@@ -9,11 +9,20 @@ module QA
                       :source_branch,
                       :target_branch
 
+        product :project do |factory|
+          factory.project
+        end
+
+        product :source_branch do |factory|
+          factory.source_branch
+        end
+
         dependency Factory::Resource::Project, as: :project do |project|
           project.name = 'project-with-merge-request'
         end
 
         dependency Factory::Repository::Push, as: :target do |push, factory|
+          factory.project.visit!
           push.project = factory.project
           push.branch_name = "master:#{factory.target_branch}"
         end

qa/qa/git/repository.rb

@@ -36,6 +36,10 @@ module QA
         `git clone #{opts} #{@uri.to_s} ./ #{suppress_output}`
       end
 
+      def checkout(branch_name)
+        `git checkout "#{branch_name}"`
+      end
+
       def shallow_clone
         clone('--depth 1')
       end

qa/qa/page/merge_request/show.rb

+module QA
+  module Page
+    module MergeRequest
+      class Show < Page::Base
+        view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js' do
+          element :merge_button
+        end
+
+        view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_merged.vue' do
+          element :merged_status, 'The changes were merged into'
+        end
+
+        view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue' do
+          element :mr_rebase_button
+          element :fast_forward_nessage, "Fast-forward merge is not possible"
+        end
+
+        def rebase!
+          wait(reload: false) do
+            click_element :mr_rebase_button
+
+            has_text?("The source branch HEAD has recently changed.")
+          end
+        end
+
+        def fast_forward_possible?
+          !has_text?("Fast-forward merge is not possible")
+        end
+
+        def has_merge_button?
+          refresh
+
+          has_selector?('.accept-merge-request')
+        end
+
+        def merge!
+          wait(reload: false) do
+            click_element :merge_button
+
+            has_text?("The changes were merged into")
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/page/project/settings/merge_request.rb

+module QA
+  module Page
+    module Project
+      module Settings
+        class MergeRequest < QA::Page::Base
+          include Common
+
+          view 'app/views/projects/_merge_request_fast_forward_settings.html.haml' do
+            element :radio_button_merge_ff
+          end
+
+          view 'app/views/projects/edit.html.haml' do
+            element :merge_request_settings, 'Merge request settings'
+            element :save_merge_request_changes
+          end
+
+          def enable_ff_only
+            expand_section('Merge request settings') do
+              click_element :radio_button_merge_ff
+              click_element :save_merge_request_changes
+            end
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/specs/features/merge_request/rebase_spec.rb

+module QA
+  feature 'merge request rebase', :core do
+    scenario 'rebases source branch of merge request'  do
+      Runtime::Browser.visit(:gitlab, Page::Main::Login)
+      Page::Main::Login.act { sign_in_using_credentials }
+
+      project = Factory::Resource::Project.fabricate! do |project|
+        project.name = "only-fast-forward"
+      end
+
+      Page::Menu::Side.act { go_to_settings }
+      Page::Project::Settings::MergeRequest.act { enable_ff_only }
+
+      merge_request = Factory::Resource::MergeRequest.fabricate! do |merge_request|
+        merge_request.project = project
+        merge_request.title = 'Needs rebasing'
+      end
+
+      Factory::Repository::Push.fabricate! do |push|
+        push.project = project
+        push.file_name = "other.txt"
+        push.file_content = "New file added!"
+      end
+
+      merge_request.visit!
+
+      Page::MergeRequest::Show.perform do |merge_request|
+        expect(merge_request).to have_content('Needs rebasing')
+        expect(merge_request).not_to be_fast_forward_possible
+        expect(merge_request).not_to have_merge_button
+
+        merge_request.rebase!
+
+        expect(merge_request).to have_merge_button
+        expect(merge_request.fast_forward_possible?).to be_truthy
+      end
+    end
+  end
+end

qa/spec/factory/base_spec.rb

@@ -7,6 +7,7 @@ describe QA::Factory::Base do
 
     before do
       allow(QA::Factory::Product).to receive(:new).and_return(product)
+      allow(QA::Factory::Product).to receive(:populate!).and_return(product)
     end
 
     it 'instantiates the factory and calls factory method' do
@@ -76,6 +77,7 @@ describe QA::Factory::Base do
         allow(subject).to receive(:new).and_return(instance)
         allow(instance).to receive(:mydep).and_return(nil)
         allow(QA::Factory::Product).to receive(:new)
+        allow(QA::Factory::Product).to receive(:populate!)
       end
 
       it 'builds all dependencies first' do
@@ -89,8 +91,16 @@ describe QA::Factory::Base do
   describe '.product' do
     subject do
       Class.new(described_class) do
+        def fabricate!
+          "any"
+        end
+
+        # Defined only to be stubbed
+        def self.find_page
+        end
+
         product :token do
-          page.do_something_on_page!
+          find_page.do_something_on_page!
           'resulting value'
         end
       end
@@ -105,16 +115,17 @@ describe QA::Factory::Base do
       let(:page) { spy('page') }
 
       before do
-        allow(subject).to receive(:new).and_return(factory)
+        allow(factory).to receive(:class).and_return(subject)
         allow(QA::Factory::Product).to receive(:new).and_return(product)
         allow(product).to receive(:page).and_return(page)
+        allow(subject).to receive(:find_page).and_return(page)
       end
 
       it 'populates product after fabrication' do
         subject.fabricate!
 
-        expect(page).to have_received(:do_something_on_page!)
         expect(product.token).to eq 'resulting value'
+        expect(page).to have_received(:do_something_on_page!)
       end
     end
   end

qa/spec/factory/product_spec.rb

 describe QA::Factory::Product do
-  let(:factory) { spy('factory') }
+  let(:factory) do
+    QA::Factory::Base.new
+  end
+
+  let(:attributes) do
+    { test: QA::Factory::Product::Attribute.new(:test, proc { 'returned' }) }
+  end
+
   let(:product) { spy('product') }
 
+  before do
+    allow(QA::Factory::Base).to receive(:attributes).and_return(attributes)
+  end
+
   describe '.populate!' do
-    it 'returns a fabrication product' do
+    it 'returns a fabrication product and define factory attributes as its methods' do
       expect(described_class).to receive(:new).and_return(product)
 
       result = described_class.populate!(factory) do |instance|
@@ -11,6 +22,7 @@ describe QA::Factory::Product do
       end
 
       expect(result).to be product
+      expect(result.test).to eq('returned')
     end
   end
 

spec/controllers/projects/pages_domains_controller_spec.rb

@@ -46,7 +46,46 @@ describe Projects::PagesDomainsController do
         post(:create, request_params.merge(pages_domain: pages_domain_params))
       end.to change { PagesDomain.count }.by(1)
 
-      expect(response).to redirect_to(project_pages_path(project))
+      created_domain = PagesDomain.reorder(:id).last
+
+      expect(created_domain).to be_present
+      expect(response).to redirect_to(project_pages_domain_path(project, created_domain))
+    end
+  end
+
+  describe 'POST verify' do
+    let(:params) { request_params.merge(id: pages_domain.domain) }
+
+    def stub_service
+      service = double(:service)
+
+      expect(VerifyPagesDomainService).to receive(:new) { service }
+
+      service
+    end
+
+    it 'handles verification success' do
+      expect(stub_service).to receive(:execute).and_return(status: :success)
+
+      post :verify, params
+
+      expect(response).to redirect_to project_pages_domain_path(project, pages_domain)
+      expect(flash[:notice]).to eq('Successfully verified domain ownership')
+    end
+
+    it 'handles verification failure' do
+      expect(stub_service).to receive(:execute).and_return(status: :failed)
+
+      post :verify, params
+
+      expect(response).to redirect_to project_pages_domain_path(project, pages_domain)
+      expect(flash[:alert]).to eq('Failed to verify domain ownership')
+    end
+
+    it 'returns a 404 response for an unknown domain' do
+      post :verify, request_params.merge(id: 'unknown-domain')
+
+      expect(response).to have_gitlab_http_status(404)
     end
   end
 

spec/controllers/projects/prometheus_controller_spec.rb → spec/controllers/projects/prometheus/metrics_controller_spec.rb

-require('spec_helper')
+require 'spec_helper'
 
-describe Projects::PrometheusController do
+describe Projects::Prometheus::MetricsController do
   let(:user) { create(:user) }
-  let!(:project) { create(:project) }
+  let(:project) { create(:project) }
 
   let(:prometheus_service) { double('prometheus_service') }
 
   before do
     allow(controller).to receive(:project).and_return(project)
-    allow(project).to receive(:prometheus_service).and_return(prometheus_service)
+    allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return(prometheus_service)
 
     project.add_master(user)
     sign_in(user)
   end
 
-  describe 'GET #active_metrics' do
+  describe 'GET #active_common' do
     context 'when prometheus metrics are enabled' do
       context 'when data is not present' do
         before do
@@ -22,7 +22,7 @@ describe Projects::PrometheusController do
         end
 
         it 'returns no content response' do
-          get :active_metrics, project_params(format: :json)
+          get :active_common, project_params(format: :json)
 
           expect(response).to have_gitlab_http_status(204)
         end
@@ -36,7 +36,7 @@ describe Projects::PrometheusController do
         end
 
         it 'returns no content response' do
-          get :active_metrics, project_params(format: :json)
+          get :active_common, project_params(format: :json)
 
           expect(response).to have_gitlab_http_status(200)
           expect(json_response).to eq(sample_response.deep_stringify_keys)
@@ -45,7 +45,7 @@ describe Projects::PrometheusController do
 
       context 'when requesting non json response' do
         it 'returns not found response' do
-          get :active_metrics, project_params
+          get :active_common, project_params
 
           expect(response).to have_gitlab_http_status(404)
         end

spec/factories/pages_domains.rb

 FactoryBot.define do
   factory :pages_domain, class: 'PagesDomain' do
-    domain 'my.domain.com'
+    sequence(:domain) { |n| "my#{n}.domain.com" }
+    verified_at { Time.now }
+    enabled_until { 1.week.from_now }
+
+    trait :disabled do
+      verified_at nil
+      enabled_until nil
+    end
+
+    trait :unverified do
+      verified_at nil
+    end
+
+    trait :reverify do
+      enabled_until { 1.hour.from_now }
+    end
+
+    trait :expired do
+      enabled_until { 1.hour.ago }
+    end
 
     trait :with_certificate do
       certificate '-----BEGIN CERTIFICATE-----

spec/features/admin/admin_runners_spec.rb

@@ -19,7 +19,7 @@ describe "Admin Runners" do
       end
 
       it 'has all necessary texts' do
-        expect(page).to have_text "How to setup"
+        expect(page).to have_text "Setup a shared Runner manually"
         expect(page).to have_text "Runners with last contact more than a minute ago: 1"
       end
 
@@ -54,7 +54,7 @@ describe "Admin Runners" do
       end
 
       it 'has all necessary texts including no runner message' do
-        expect(page).to have_text "How to setup"
+        expect(page).to have_text "Setup a shared Runner manually"
         expect(page).to have_text "Runners with last contact more than a minute ago: 0"
         expect(page).to have_text 'No runners found'
       end

spec/features/issues/filtered_search/recent_searches_spec.rb

@@ -39,8 +39,8 @@ describe 'Recent searches', :js do
 
     items = all('.filtered-search-history-dropdown-item', visible: false, count: 2)
 
-    expect(items[0].text).to eq('label:~qux garply')
-    expect(items[1].text).to eq('label:~foo bar')
+    expect(items[0].text).to eq('label: ~qux garply')
+    expect(items[1].text).to eq('label: ~foo bar')
   end
 
   it 'saved recent searches are restored last on the list' do

spec/features/projects/pages_spec.rb

@@ -60,7 +60,6 @@ feature 'Pages' do
             fill_in 'Domain', with: 'my.test.domain.com'
             click_button 'Create New Domain'
 
-            expect(page).to have_content('Domains (1)')
             expect(page).to have_content('my.test.domain.com')
           end
         end
@@ -159,7 +158,6 @@ feature 'Pages' do
           fill_in 'Key (PEM)', with: certificate_key
           click_button 'Create New Domain'
 
-          expect(page).to have_content('Domains (1)')
           expect(page).to have_content('my.test.domain.com')
         end
       end

spec/features/runners_spec.rb

@@ -7,6 +7,20 @@ feature 'Runners' do
     sign_in(user)
   end
 
+  context 'when user opens runners page' do
+    given(:project) { create(:project) }
+
+    background do
+      project.add_master(user)
+    end
+
+    scenario 'user can see a button to install runners on kubernetes clusters' do
+      visit runners_path(project)
+
+      expect(page).to have_link('Install Runner on Kubernetes', href: project_clusters_path(project))
+    end
+  end
+
   context 'when a project has enabled shared_runners' do
     given(:project) { create(:project) }
 

spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json

@@ -4,6 +4,9 @@
     "domain": { "type": "string" },
     "url": { "type": "uri" },
     "project_id": { "type": "integer" },
+    "verified": { "type": "boolean" },
+    "verification_code": { "type": ["string", "null"] },
+    "enabled_until": { "type": ["date", "null"] },
     "certificate_expiration": {
       "type": "object",
       "properties": {
@@ -14,6 +17,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url", "project_id"],
+  "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until"],
   "additionalProperties": false
 }

spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json

@@ -3,6 +3,9 @@
   "properties": {
     "domain": { "type": "string" },
     "url": { "type": "uri" },
+    "verified": { "type": "boolean" },
+    "verification_code": { "type": ["string", "null"] },
+    "enabled_until": { "type": ["date", "null"] },
     "certificate": {
       "type": "object",
       "properties": {
@@ -15,6 +18,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url"],
+  "required": ["domain", "url", "verified", "verification_code", "enabled_until"],
   "additionalProperties": false
 }

spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js

 import Vue from 'vue';
 import eventHub from '~/filtered_search/event_hub';
-import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content';
-
+import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content.vue';
 import FilteredSearchTokenKeys from '~/filtered_search/filtered_search_token_keys';
 
 const createComponent = (propsData) => {

spec/javascripts/test_bundle.js

@@ -113,7 +113,9 @@ if (process.env.BABEL_ENV === 'coverage') {
   // exempt these files from the coverage report
   const troubleMakers = [
     './blob_edit/blob_bundle.js',
-    './boards/boards_bundle.js',
+    './boards/components/modal/empty_state.js',
+    './boards/components/modal/footer.js',
+    './boards/components/modal/header.js',
     './cycle_analytics/cycle_analytics_bundle.js',
     './cycle_analytics/components/stage_plan_component.js',
     './cycle_analytics/components/stage_staging_component.js',

spec/lib/gitlab/git_access_spec.rb

@@ -534,6 +534,19 @@ describe Gitlab::GitAccess do
       expect { pull_access_check }.to raise_unauthorized('Your account has been blocked.')
     end
 
+    context 'when the project repository does not exist' do
+      it 'returns not found' do
+        project.add_guest(user)
+        repo = project.repository
+        FileUtils.rm_rf(repo.path)
+
+        # Sanity check for rm_rf
+        expect(repo.exists?).to eq(false)
+
+        expect { pull_access_check }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.')
+      end
+    end
+
     describe 'without access to project' do
       context 'pull code' do
         it { expect { pull_access_check }.to raise_not_found }

spec/lib/gitlab/git_access_wiki_spec.rb

@@ -57,7 +57,7 @@ describe Gitlab::GitAccessWiki do
           # Sanity check for rm_rf
           expect(wiki_repo.exists?).to eq(false)
 
-          expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'A repository for this project does not exist yet.')
+          expect { subject }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.')
         end
       end
     end

spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb

@@ -24,7 +24,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
 
   context 'with one group where two metrics is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
       allow(client).to receive(:label_values).and_return(metric_names)
     end
 
@@ -70,7 +70,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
 
   context 'with one group where only one metric is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
       allow(client).to receive(:label_values).and_return('metric_a')
     end
 
@@ -99,7 +99,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do
     let(:second_metric_group) { simple_metric_group(name: 'nameb', metrics: simple_metrics(added_metric_name: 'metric_c')) }
 
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group, second_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group, second_metric_group])
       allow(client).to receive(:label_values).and_return('metric_c')
     end
 

spec/lib/gitlab/prometheus_client_spec.rb

@@ -19,41 +19,41 @@ describe Gitlab::PrometheusClient do
   # - execute_query: A query call
   shared_examples 'failure response' do
     context 'when request returns 400 with an error message' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 400, body: { error: 'bar!' })
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'bar!')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'bar!')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns 400 without an error message' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 400)
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'Bad data received')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'Bad data received')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns 500' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 500, body: { message: 'FAIL!' })
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, '500 - {"message":"FAIL!"}')
+          .to raise_error(Gitlab::PrometheusClient::Error, '500 - {"message":"FAIL!"}')
         expect(req_stub).to have_been_requested
       end
     end
 
     context 'when request returns non json data' do
-      it 'raises a Gitlab::PrometheusError error' do
+      it 'raises a Gitlab::PrometheusClient::Error error' do
         req_stub = stub_prometheus_request(query_url, status: 200, body: 'not json')
 
         expect { execute_query }
-          .to raise_error(Gitlab::PrometheusError, 'Parsing response failed')
+          .to raise_error(Gitlab::PrometheusClient::Error, 'Parsing response failed')
         expect(req_stub).to have_been_requested
       end
     end
@@ -65,27 +65,27 @@ describe Gitlab::PrometheusClient do
     subject { described_class.new(RestClient::Resource.new(prometheus_url)) }
 
     context 'exceptions are raised' do
-      it 'raises a Gitlab::PrometheusError error when a SocketError is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a SocketError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, SocketError)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "Can't connect to #{prometheus_url}")
+          .to raise_error(Gitlab::PrometheusClient::Error, "Can't connect to #{prometheus_url}")
         expect(req_stub).to have_been_requested
       end
 
-      it 'raises a Gitlab::PrometheusError error when a SSLError is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a SSLError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, OpenSSL::SSL::SSLError)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "#{prometheus_url} contains invalid SSL data")
+          .to raise_error(Gitlab::PrometheusClient::Error, "#{prometheus_url} contains invalid SSL data")
         expect(req_stub).to have_been_requested
       end
 
-      it 'raises a Gitlab::PrometheusError error when a RestClient::Exception is rescued' do
+      it 'raises a Gitlab::PrometheusClient::Error error when a RestClient::Exception is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, RestClient::Exception)
 
         expect { subject.send(:get, '/', {}) }
-          .to raise_error(Gitlab::PrometheusError, "Network connection error")
+          .to raise_error(Gitlab::PrometheusClient::Error, "Network connection error")
         expect(req_stub).to have_been_requested
       end
     end

spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb → spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb

 require 'spec_helper'
 
-describe Gitlab::SidekiqMiddleware::MemoryKiller do
+describe Gitlab::SidekiqMiddleware::Shutdown do
   subject { described_class.new }
-  let(:pid) { 999 }
 
+  let(:pid) { Process.pid }
   let(:worker) { double(:worker, class: 'TestWorker') }
   let(:job) { { 'jid' => 123 } }
   let(:queue) { 'test_queue' }
+  let(:block) { proc { nil } }
 
   def run
-    thread = subject.call(worker, job, queue) { nil }
-    thread&.join
+    subject.call(worker, job, queue) { block.call }
+    described_class.shutdown_thread&.join
+  end
+
+  def pop_trace
+    subject.trace.pop(true)
   end
 
   before do
     allow(subject).to receive(:get_rss).and_return(10.kilobytes)
-    allow(subject).to receive(:pid).and_return(pid)
+    described_class.clear_shutdown_thread
   end
 
   context 'when MAX_RSS is set to 0' do
@@ -30,22 +35,26 @@ describe Gitlab::SidekiqMiddleware::MemoryKiller do
     end
   end
 
+  def expect_shutdown_sequence
+    expect(pop_trace).to eq([:sleep, 15 * 60])
+    expect(pop_trace).to eq([:kill, 'SIGTSTP', pid])
+
+    expect(pop_trace).to eq([:sleep, 30])
+    expect(pop_trace).to eq([:kill, 'SIGTERM', pid])
+
+    expect(pop_trace).to eq([:sleep, 10])
+    expect(pop_trace).to eq([:kill, 'SIGKILL', pid])
+  end
+
   context 'when MAX_RSS is exceeded' do
     before do
       stub_const("#{described_class}::MAX_RSS", 5.kilobytes)
     end
 
-    it 'sends the STP, TERM and KILL signals at expected times' do
-      expect(subject).to receive(:sleep).with(15 * 60).ordered
-      expect(Process).to receive(:kill).with('SIGSTP', pid).ordered
-
-      expect(subject).to receive(:sleep).with(30).ordered
-      expect(Process).to receive(:kill).with('SIGTERM', pid).ordered
-
-      expect(subject).to receive(:sleep).with(10).ordered
-      expect(Process).to receive(:kill).with('SIGKILL', pid).ordered
-
+    it 'sends the TSTP, TERM and KILL signals at expected times' do
       run
+
+      expect_shutdown_sequence
     end
   end
 
@@ -60,4 +69,20 @@ describe Gitlab::SidekiqMiddleware::MemoryKiller do
       run
     end
   end
+
+  context 'when WantShutdown is raised' do
+    let(:block) { proc { raise described_class::WantShutdown } }
+
+    it 'starts the shutdown sequence and re-raises the exception' do
+      expect { run }.to raise_exception(described_class::WantShutdown)
+
+      # We can't expect 'run' to have joined on the shutdown thread, because
+      # it hit an exception.
+      shutdown_thread = described_class.shutdown_thread
+      expect(shutdown_thread).not_to be_nil
+      shutdown_thread.join
+
+      expect_shutdown_sequence
+    end
+  end
 end

spec/mailers/emails/pages_domains_spec.rb

+require 'spec_helper'
+require 'email_spec'
+
+describe Emails::PagesDomains do
+  include EmailSpec::Matchers
+  include_context 'gitlab email notification'
+
+  set(:project) { create(:project) }
+  set(:domain) { create(:pages_domain, project: project)  }
+  set(:user) { project.owner }
+
+  shared_examples 'a pages domain email' do
+    it_behaves_like 'an email sent from GitLab'
+    it_behaves_like 'it should not have Gmail Actions links'
+    it_behaves_like 'a user cannot unsubscribe through footer link'
+
+    it 'has the expected content' do
+      aggregate_failures do
+        is_expected.to have_subject(email_subject)
+        is_expected.to have_body_text(project.human_name)
+        is_expected.to have_body_text(domain.domain)
+        is_expected.to have_body_text domain.url
+        is_expected.to have_body_text project_pages_domain_url(project, domain)
+        is_expected.to have_body_text help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+      end
+    end
+  end
+
+  describe '#pages_domain_enabled_email' do
+    let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been enabled" }
+
+    subject { Notify.pages_domain_enabled_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'has been enabled' }
+  end
+
+  describe '#pages_domain_disabled_email' do
+    let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been disabled" }
+
+    subject { Notify.pages_domain_disabled_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'has been disabled' }
+  end
+
+  describe '#pages_domain_verification_succeeded_email' do
+    let(:email_subject) { "#{project.path} | Verification succeeded for GitLab Pages domain '#{domain.domain}'" }
+
+    subject { Notify.pages_domain_verification_succeeded_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it { is_expected.to have_body_text 'successfully verified' }
+  end
+
+  describe '#pages_domain_verification_failed_email' do
+    let(:email_subject) { "#{project.path} | ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'" }
+
+    subject { Notify.pages_domain_verification_failed_email(domain, user) }
+
+    it_behaves_like 'a pages domain email'
+
+    it 'says verification has failed and when the domain is enabled until' do
+      is_expected.to have_body_text 'Verification has failed'
+      is_expected.to have_body_text domain.enabled_until.strftime('%F %T')
+    end
+  end
+end

spec/migrations/enqueue_verify_pages_domain_workers_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180216121030_enqueue_verify_pages_domain_workers')
+
+describe EnqueueVerifyPagesDomainWorkers, :sidekiq, :migration do
+  around do |example|
+    Sidekiq::Testing.fake! do
+      example.run
+    end
+  end
+
+  describe '#up' do
+    it 'enqueues a verification worker for every domain' do
+      domains = 1.upto(3).map { |i| PagesDomain.create!(domain: "my#{i}.domain.com") }
+
+      expect { migrate! }.to change(PagesDomainVerificationWorker.jobs, :size).by(3)
+
+      enqueued_ids = PagesDomainVerificationWorker.jobs.map { |job| job['args'] }
+      expected_ids = domains.map { |domain| [domain.id] }
+
+      expect(enqueued_ids).to match_array(expected_ids)
+    end
+  end
+end

spec/models/pages_domain_spec.rb

 require 'spec_helper'
 
 describe PagesDomain do
+  using RSpec::Parameterized::TableSyntax
+
+  subject(:pages_domain) { described_class.new }
+
   describe 'associations' do
     it { is_expected.to belong_to(:project) }
   end
@@ -64,19 +68,51 @@ describe PagesDomain do
     end
   end
 
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:verification_code) }
+  end
+
+  describe '#verification_code' do
+    subject { pages_domain.verification_code }
+
+    it 'is set automatically with 128 bits of SecureRandom data' do
+      expect(SecureRandom).to receive(:hex).with(16) { 'verification code' }
+
+      is_expected.to eq('verification code')
+    end
+  end
+
+  describe '#keyed_verification_code' do
+    subject { pages_domain.keyed_verification_code }
+
+    it { is_expected.to eq("gitlab-pages-verification-code=#{pages_domain.verification_code}") }
+  end
+
+  describe '#verification_domain' do
+    subject { pages_domain.verification_domain }
+
+    it { is_expected.to be_nil }
+
+    it 'is a well-known subdomain if the domain is present' do
+      pages_domain.domain = 'example.com'
+
+      is_expected.to eq('_gitlab-pages-verification-code.example.com')
+    end
+  end
+
   describe '#url' do
     subject { domain.url }
 
     context 'without the certificate' do
       let(:domain) { build(:pages_domain, certificate: '') }
 
-      it { is_expected.to eq('http://my.domain.com') }
+      it { is_expected.to eq("http://#{domain.domain}") }
     end
 
     context 'with a certificate' do
       let(:domain) { build(:pages_domain, :with_certificate) }
 
-      it { is_expected.to eq('https://my.domain.com') }
+      it { is_expected.to eq("https://#{domain.domain}") }
     end
   end
 
@@ -154,4 +190,108 @@ describe PagesDomain do
     # We test only existence of output, since the output is long
     it { is_expected.not_to be_empty }
   end
+
+  describe '#update_daemon' do
+    it 'runs when the domain is created' do
+      domain = build(:pages_domain)
+
+      expect(domain).to receive(:update_daemon)
+
+      domain.save!
+    end
+
+    it 'runs when the domain is destroyed' do
+      domain = create(:pages_domain)
+
+      expect(domain).to receive(:update_daemon)
+
+      domain.destroy!
+    end
+
+    it 'delegates to Projects::UpdatePagesConfigurationService' do
+      service = instance_double('Projects::UpdatePagesConfigurationService')
+      expect(Projects::UpdatePagesConfigurationService).to receive(:new) { service }
+      expect(service).to receive(:execute)
+
+      create(:pages_domain)
+    end
+
+    context 'configuration updates when attributes change' do
+      set(:project1) { create(:project) }
+      set(:project2) { create(:project) }
+      set(:domain) { create(:pages_domain) }
+
+      where(:attribute, :old_value, :new_value, :update_expected) do
+        now = Time.now
+        future = now + 1.day
+
+        :project | nil       | :project1 | true
+        :project | :project1 | :project1 | false
+        :project | :project1 | :project2 | true
+        :project | :project1 | nil       | true
+
+        # domain can't be set to nil
+        :domain | 'a.com' | 'a.com' | false
+        :domain | 'a.com' | 'b.com' | true
+
+        # verification_code can't be set to nil
+        :verification_code | 'foo' | 'foo'  | false
+        :verification_code | 'foo' | 'bar'  | false
+
+        :verified_at | nil | now    | false
+        :verified_at | now | now    | false
+        :verified_at | now | future | false
+        :verified_at | now | nil    | false
+
+        :enabled_until | nil | now    | true
+        :enabled_until | now | now    | false
+        :enabled_until | now | future | false
+        :enabled_until | now | nil    | true
+      end
+
+      with_them do
+        it 'runs if a relevant attribute has changed' do
+          a = old_value.is_a?(Symbol) ? send(old_value) : old_value
+          b = new_value.is_a?(Symbol) ? send(new_value) : new_value
+
+          domain.update!(attribute => a)
+
+          if update_expected
+            expect(domain).to receive(:update_daemon)
+          else
+            expect(domain).not_to receive(:update_daemon)
+          end
+
+          domain.update!(attribute => b)
+        end
+      end
+
+      context 'TLS configuration' do
+        set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) }
+
+        let(:cert1) { domain_with_tls.certificate }
+        let(:cert2) { cert1 + ' ' }
+        let(:key1) { domain_with_tls.key }
+        let(:key2) { key1 + ' ' }
+
+        it 'updates when added' do
+          expect(domain).to receive(:update_daemon)
+
+          domain.update!(key: key1, certificate: cert1)
+        end
+
+        it 'updates when changed' do
+          expect(domain_with_tls).to receive(:update_daemon)
+
+          domain_with_tls.update!(key: key2, certificate: cert2)
+        end
+
+        it 'updates when removed' do
+          expect(domain_with_tls).to receive(:update_daemon)
+
+          domain_with_tls.update!(key: nil, certificate: nil)
+        end
+      end
+    end
+  end
 end

spec/models/project_services/prometheus_service_spec.rb

@@ -223,8 +223,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
 
       context 'with cluster for all environments without prometheus installed' do
         context 'without environment supplied' do
-          it 'raises PrometheusError because cluster was not found' do
-            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+          it 'raises PrometheusClient::Error because cluster was not found' do
+            expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/)
           end
         end
 
@@ -242,8 +242,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
         context 'with prod environment supplied' do
           let!(:environment) { create(:environment, project: project, name: 'prod') }
 
-          it 'raises PrometheusError because cluster was not found' do
-            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+          it 'raises PrometheusClient::Error because cluster was not found' do
+            expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/)
           end
         end
       end

spec/requests/git_http_spec.rb

@@ -597,7 +597,7 @@ describe 'Git HTTP requests' do
         context "when a gitlab ci token is provided" do
           let(:project) { create(:project, :repository) }
           let(:build) { create(:ci_build, :running) }
-          let(:other_project) { create(:project) }
+          let(:other_project) { create(:project, :repository) }
 
           before do
             build.update!(project: project) # can't associate it on factory create
@@ -648,10 +648,10 @@ describe 'Git HTTP requests' do
               context 'when the repo does not exist' do
                 let(:project) { create(:project) }
 
-                it 'rejects pulls with 403 Forbidden' do
+                it 'rejects pulls with 404 Not Found' do
                   clone_get path, env
 
-                  expect(response).to have_gitlab_http_status(:forbidden)
+                  expect(response).to have_gitlab_http_status(:not_found)
                   expect(response.body).to eq(git_access_error(:no_repo))
                 end
               end

spec/services/ci/create_trace_artifact_service_spec.rb

@@ -4,40 +4,60 @@ describe Ci::CreateTraceArtifactService do
   describe '#execute' do
     subject { described_class.new(nil, nil).execute(job) }
 
-    let(:job) { create(:ci_build) }
-
     context 'when the job does not have trace artifact' do
       context 'when the job has a trace file' do
-        before do
-          allow_any_instance_of(Gitlab::Ci::Trace)
-            .to receive(:default_path) { expand_fixture_path('trace/sample_trace') }
+        let!(:job) { create(:ci_build, :trace_live) }
+        let!(:legacy_path) { job.trace.read { |stream| return stream.path } }
+        let!(:legacy_checksum) { Digest::SHA256.file(legacy_path).hexdigest }
+        let(:new_path) { job.job_artifacts_trace.file.path }
+        let(:new_checksum) { Digest::SHA256.file(new_path).hexdigest }
 
-          allow_any_instance_of(JobArtifactUploader).to receive(:move_to_cache) { false }
-          allow_any_instance_of(JobArtifactUploader).to receive(:move_to_store) { false }
-        end
+        it { expect(File.exist?(legacy_path)).to be_truthy }
 
         it 'creates trace artifact' do
           expect { subject }.to change { Ci::JobArtifact.count }.by(1)
 
-          expect(job.job_artifacts_trace.read_attribute(:file)).to eq('sample_trace')
+          expect(File.exist?(legacy_path)).to be_falsy
+          expect(File.exist?(new_path)).to be_truthy
+          expect(new_checksum).to eq(legacy_checksum)
+          expect(job.job_artifacts_trace.file.exists?).to be_truthy
+          expect(job.job_artifacts_trace.file.filename).to eq('job.log')
         end
 
-        context 'when the job has already had trace artifact' do
+        context 'when failed to create trace artifact record' do
           before do
-            create(:ci_job_artifact, :trace, job: job)
+            # When ActiveRecord error happens
+            allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false)
+            allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages)
+              .and_return("Error")
+
+            subject rescue nil
+
+            job.reload
           end
 
-          it 'does not create trace artifact' do
-            expect { subject }.not_to change { Ci::JobArtifact.count }
+          it 'keeps legacy trace and removes trace artifact' do
+            expect(File.exist?(legacy_path)).to be_truthy
+            expect(job.job_artifacts_trace).to be_nil
           end
         end
       end
 
       context 'when the job does not have a trace file' do
+        let!(:job) { create(:ci_build) }
+
         it 'does not create trace artifact' do
           expect { subject }.not_to change { Ci::JobArtifact.count }
         end
       end
     end
+
+    context 'when the job has already had trace artifact' do
+      let!(:job) { create(:ci_build, :trace_artifact) }
+
+      it 'does not create trace artifact' do
+        expect { subject }.not_to change { Ci::JobArtifact.count }
+      end
+    end
   end
 end

spec/services/notification_service_spec.rb

@@ -1678,6 +1678,78 @@ describe NotificationService, :mailer do
     end
   end
 
+  describe 'Pages domains' do
+    set(:project) { create(:project) }
+    set(:domain) { create(:pages_domain, project: project) }
+    set(:u_blocked) { create(:user, :blocked) }
+    set(:u_silence) { create_user_with_notification(:disabled, 'silent', project) }
+    set(:u_owner)   { project.owner }
+    set(:u_master1) { create(:user) }
+    set(:u_master2) { create(:user) }
+    set(:u_developer) { create(:user) }
+
+    before do
+      project.add_master(u_blocked)
+      project.add_master(u_silence)
+      project.add_master(u_master1)
+      project.add_master(u_master2)
+      project.add_developer(u_developer)
+
+      reset_delivered_emails!
+    end
+
+    %i[
+      pages_domain_enabled
+      pages_domain_disabled
+      pages_domain_verification_succeeded
+      pages_domain_verification_failed
+    ].each do |sym|
+      describe "##{sym}" do
+        subject(:notify!) { notification.send(sym, domain) }
+
+        it 'emails current watching masters' do
+          expect(Notify).to receive(:"#{sym}_email").at_least(:once).and_call_original
+
+          notify!
+
+          should_only_email(u_master1, u_master2, u_owner)
+        end
+
+        it 'emails nobody if the project is missing' do
+          domain.project = nil
+
+          notify!
+
+          should_not_email_anyone
+        end
+      end
+    end
+
+    describe '#pages_domain_verification_failed' do
+      it 'emails current watching masters' do
+        notification.pages_domain_verification_failed(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+
+    describe '#pages_domain_enabled' do
+      it 'emails current watching masters' do
+        notification.pages_domain_enabled(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+
+    describe '#pages_domain_disabled' do
+      it 'emails current watching masters' do
+        notification.pages_domain_disabled(domain)
+
+        should_only_email(u_master1, u_master2, u_owner)
+      end
+    end
+  end
+
   def build_team(project)
     @u_watcher               = create_global_setting_for(create(:user), :watch)
     @u_participating         = create_global_setting_for(create(:user), :participating)

spec/services/verify_pages_domain_service_spec.rb

+require 'spec_helper'
+
+describe VerifyPagesDomainService do
+  using RSpec::Parameterized::TableSyntax
+  include EmailHelpers
+
+  let(:error_status) { { status: :error, message: "Couldn't verify #{domain.domain}" } }
+
+  subject(:service) { described_class.new(domain) }
+
+  describe '#execute' do
+    context 'verification code recognition (verified domain)' do
+      where(:domain_sym, :code_sym) do
+        :domain | :verification_code
+        :domain | :keyed_verification_code
+
+        :verification_domain | :verification_code
+        :verification_domain | :keyed_verification_code
+      end
+
+      with_them do
+        set(:domain) { create(:pages_domain) }
+
+        let(:domain_name) { domain.send(domain_sym) }
+        let(:verification_code) { domain.send(code_sym) }
+
+        it 'verifies and enables the domain' do
+          stub_resolver(domain_name => ['something else', verification_code])
+
+          expect(service.execute).to eq(status: :success)
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'verifies and enables when the code is contained partway through a TXT record' do
+          stub_resolver(domain_name => "something #{verification_code} else")
+
+          expect(service.execute).to eq(status: :success)
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'does not verify when the code is not present' do
+          stub_resolver(domain_name => 'something else')
+
+          expect(service.execute).to eq(error_status)
+
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+      end
+
+      context 'verified domain' do
+        set(:domain) { create(:pages_domain) }
+
+        it 'unverifies (but does not disable) when the right code is not present' do
+          stub_resolver(domain.domain => 'something else')
+
+          expect(service.execute).to eq(error_status)
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'unverifies (but does not disable) when no records are present' do
+          stub_resolver
+
+          expect(service.execute).to eq(error_status)
+          expect(domain).not_to be_verified
+          expect(domain).to be_enabled
+        end
+      end
+
+      context 'expired domain' do
+        set(:domain) { create(:pages_domain, :expired) }
+
+        it 'verifies and enables when the right code is present' do
+          stub_resolver(domain.domain => domain.keyed_verification_code)
+
+          expect(service.execute).to eq(status: :success)
+
+          expect(domain).to be_verified
+          expect(domain).to be_enabled
+        end
+
+        it 'disables when the right code is not present' do
+          error_status[:message] += '. It is now disabled.'
+
+          stub_resolver
+
+          expect(service.execute).to eq(error_status)
+
+          expect(domain).not_to be_verified
+          expect(domain).not_to be_enabled
+        end
+      end
+    end
+
+    context 'timeout behaviour' do
+      let(:domain) { create(:pages_domain) }
+
+      it 'sets a timeout on the DNS query' do
+        expect(stub_resolver).to receive(:timeouts=).with(described_class::RESOLVER_TIMEOUT_SECONDS)
+
+        service.execute
+      end
+    end
+
+    context 'email notifications' do
+      let(:notification_service) { instance_double('NotificationService') }
+
+      where(:factory, :verification_succeeds, :expected_notification) do
+        nil         | true  | nil
+        nil         | false | :verification_failed
+        :reverify   | true  | nil
+        :reverify   | false | :verification_failed
+        :unverified | true  | :verification_succeeded
+        :unverified | false | nil
+        :expired    | true  | nil
+        :expired    | false | :disabled
+        :disabled   | true  | :enabled
+        :disabled   | false | nil
+      end
+
+      with_them do
+        let(:domain) { create(:pages_domain, *[factory].compact) }
+
+        before do
+          allow(service).to receive(:notification_service) { notification_service }
+
+          if verification_succeeds
+            stub_resolver(domain.domain => domain.verification_code)
+          else
+            stub_resolver
+          end
+        end
+
+        it 'sends a notification if appropriate' do
+          if expected_notification
+            expect(notification_service).to receive(:"pages_domain_#{expected_notification}").with(domain)
+          end
+
+          service.execute
+        end
+      end
+
+      context 'pages verification disabled' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        before do
+          stub_application_setting(pages_domain_verification_enabled: false)
+          allow(service).to receive(:notification_service) { notification_service }
+        end
+
+        it 'skips email notifications' do
+          expect(notification_service).not_to receive(:pages_domain_enabled)
+
+          service.execute
+        end
+      end
+    end
+
+    context 'pages configuration updates' do
+      context 'enabling a disabled domain' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        it 'schedules an update' do
+          stub_resolver(domain.domain => domain.verification_code)
+
+          expect(domain).to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'verifying an enabled domain' do
+        let(:domain) { create(:pages_domain) }
+
+        it 'schedules an update' do
+          stub_resolver(domain.domain => domain.verification_code)
+
+          expect(domain).not_to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'disabling an expired domain' do
+        let(:domain) { create(:pages_domain, :expired) }
+
+        it 'schedules an update' do
+          stub_resolver
+
+          expect(domain).to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+
+      context 'failing to verify a disabled domain' do
+        let(:domain) { create(:pages_domain, :disabled) }
+
+        it 'does not schedule an update' do
+          stub_resolver
+
+          expect(domain).not_to receive(:update_daemon)
+
+          service.execute
+        end
+      end
+    end
+
+    context 'no verification code' do
+      let(:domain) { create(:pages_domain) }
+
+      it 'returns an error' do
+        domain.verification_code = ''
+
+        disallow_resolver!
+
+        expect(service.execute).to eq(status: :error, message: "No verification code set for #{domain.domain}")
+      end
+    end
+
+    context 'pages domain verification is disabled' do
+      let(:domain) { create(:pages_domain, :disabled) }
+
+      before do
+        stub_application_setting(pages_domain_verification_enabled: false)
+      end
+
+      it 'extends domain validity by unconditionally reverifying' do
+        disallow_resolver!
+
+        service.execute
+
+        expect(domain).to be_verified
+        expect(domain).to be_enabled
+      end
+
+      it 'does not shorten any grace period' do
+        grace = Time.now + 1.year
+        domain.update!(enabled_until: grace)
+        disallow_resolver!
+
+        service.execute
+
+        expect(domain.enabled_until).to be_like_time(grace)
+      end
+    end
+  end
+
+  def disallow_resolver!
+    expect(Resolv::DNS).not_to receive(:open)
+  end
+
+  def stub_resolver(stubbed_lookups = {})
+    resolver = instance_double('Resolv::DNS')
+    allow(resolver).to receive(:timeouts=)
+
+    expect(Resolv::DNS).to receive(:open).and_yield(resolver)
+
+    allow(resolver).to receive(:getresources) { [] }
+    stubbed_lookups.each do |domain, records|
+      records = Array(records).map { |txt| Resolv::DNS::Resource::IN::TXT.new(txt) }
+      allow(resolver).to receive(:getresources).with(domain, Resolv::DNS::Resource::IN::TXT) { records }
+    end
+
+    resolver
+  end
+end

spec/support/capybara.rb

@@ -78,8 +78,10 @@ RSpec.configure do |config|
   end
 
   config.after(:example, :js) do |example|
-    # prevent localstorage from introducing side effects based on test order
+    # prevent localStorage from introducing side effects based on test order
+    unless ['', 'about:blank', 'data:,'].include? Capybara.current_session.driver.browser.current_url
       execute_script("localStorage.clear();")
+    end
 
     # capybara/rspec already calls Capybara.reset_sessions! in an `after` hook,
     # but `block_and_wait_for_requests_complete` is called before it so by

spec/support/db_cleaner.rb

@@ -28,11 +28,11 @@ RSpec.configure do |config|
   end
 
   config.before(:each, :js) do
-    DatabaseCleaner.strategy = :deletion
+    DatabaseCleaner.strategy = :deletion, { cache_tables: false }
   end
 
   config.before(:each, :delete) do
-    DatabaseCleaner.strategy = :deletion
+    DatabaseCleaner.strategy = :deletion, { cache_tables: false }
   end
 
   config.before(:each, :migration) do

spec/support/prometheus/additional_metrics_shared_examples.rb

@@ -12,11 +12,12 @@ RSpec.shared_examples 'additional metrics query' do
 
   let(:client) { double('prometheus_client') }
   let(:query_result) { described_class.new(client).query(*query_params) }
-  let(:environment) { create(:environment, slug: 'environment-slug') }
+  let(:project) { create(:project) }
+  let(:environment) { create(:environment, slug: 'environment-slug', project: project) }
 
   before do
     allow(client).to receive(:label_values).and_return(metric_names)
-    allow(metric_group_class).to receive(:all).and_return([simple_metric_group(metrics: [simple_metric])])
+    allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group(metrics: [simple_metric])])
   end
 
   context 'metrics query context' do
@@ -24,13 +25,14 @@ RSpec.shared_examples 'additional metrics query' do
 
     shared_examples 'query context containing environment slug and filter' do
       it 'contains ci_environment_slug' do
-        expect(subject).to receive(:query_metrics).with(hash_including(ci_environment_slug: environment.slug))
+        expect(subject).to receive(:query_metrics).with(project, hash_including(ci_environment_slug: environment.slug))
 
         subject.query(*query_params)
       end
 
       it 'contains environment filter' do
         expect(subject).to receive(:query_metrics).with(
+          project,
           hash_including(
             environment_filter: "container_name!=\"POD\",environment=\"#{environment.slug}\""
           )
@@ -48,7 +50,7 @@ RSpec.shared_examples 'additional metrics query' do
         it_behaves_like 'query context containing environment slug and filter'
 
         it 'query context contains kube_namespace' do
-          expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace))
+          expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: kube_namespace))
 
           subject.query(*query_params)
         end
@@ -72,7 +74,7 @@ RSpec.shared_examples 'additional metrics query' do
       it_behaves_like 'query context containing environment slug and filter'
 
       it 'query context contains empty kube_namespace' do
-        expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: ''))
+        expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: ''))
 
         subject.query(*query_params)
       end
@@ -81,7 +83,7 @@ RSpec.shared_examples 'additional metrics query' do
 
   context 'with one group where two metrics is found' do
     before do
-      allow(metric_group_class).to receive(:all).and_return([simple_metric_group])
+      allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group])
     end
 
     context 'some queries return results' do
@@ -117,7 +119,7 @@ RSpec.shared_examples 'additional metrics query' do
     let(:metrics) { [simple_metric(queries: [simple_query])] }
 
     before do
-      allow(metric_group_class).to receive(:all).and_return(
+      allow(metric_group_class).to receive(:common_metrics).and_return(
         [
           simple_metric_group(name: 'group_a', metrics: [simple_metric(queries: [simple_query])]),
           simple_metric_group(name: 'group_b', metrics: [simple_metric(title: 'title_b', queries: [simple_query('b')])])

spec/workers/pages_domain_verification_cron_worker_spec.rb

+require 'spec_helper'
+
+describe PagesDomainVerificationCronWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    it 'enqueues a PagesDomainVerificationWorker for domains needing verification' do
+      verified = create(:pages_domain)
+      reverify = create(:pages_domain, :reverify)
+      disabled = create(:pages_domain, :disabled)
+
+      [reverify, disabled].each do |domain|
+        expect(PagesDomainVerificationWorker).to receive(:perform_async).with(domain.id)
+      end
+
+      expect(PagesDomainVerificationWorker).not_to receive(:perform_async).with(verified.id)
+
+      worker.perform
+    end
+  end
+end

spec/workers/pages_domain_verification_worker_spec.rb

+require 'spec_helper'
+
+describe PagesDomainVerificationWorker do
+  subject(:worker) { described_class.new }
+
+  let(:domain) { create(:pages_domain) }
+
+  describe '#perform' do
+    it 'does nothing for a non-existent domain' do
+      domain.destroy
+
+      expect(VerifyPagesDomainService).not_to receive(:new)
+
+      expect { worker.perform(domain.id) }.not_to raise_error
+    end
+
+    it 'delegates to VerifyPagesDomainService' do
+      service = double(:service)
+      expected_domain = satisfy { |obj| obj == domain }
+
+      expect(VerifyPagesDomainService).to receive(:new).with(expected_domain) { service }
+      expect(service).to receive(:execute)
+
+      worker.perform(domain.id)
+    end
+  end
+end

spec/workers/stuck_import_jobs_worker_spec.rb

@@ -2,35 +2,59 @@ require 'spec_helper'
 
 describe StuckImportJobsWorker do
   let(:worker) { described_class.new }
-  let(:exclusive_lease_uuid) { SecureRandom.uuid }
 
+  shared_examples 'project import job detection' do
+    context 'when the job has completed' do
+      context 'when the import status was already updated' do
         before do
-    allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid)
+          allow(Gitlab::SidekiqStatus).to receive(:completed_jids) do
+            project.import_start
+            project.import_finish
+
+            [project.import_jid]
+          end
         end
 
-  describe 'with started import_status' do
-    let(:project) { create(:project, :import_started, import_jid: '123') }
+        it 'does not mark the project as failed' do
+          worker.perform
+
+          expect(project.reload.import_status).to eq('finished')
+        end
+      end
+
+      context 'when the import status was not updated' do
+        before do
+          allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([project.import_jid])
+        end
 
-    describe 'long running import' do
         it 'marks the project as failed' do
-        allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return(['123'])
+          worker.perform
 
-        expect { worker.perform }.to change { project.reload.import_status }.to('failed')
+          expect(project.reload.import_status).to eq('failed')
+        end
       end
     end
 
-    describe 'running import' do
-      it 'does not mark the project as failed' do
+    context 'when the job is still in Sidekiq' do
+      before do
         allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([])
+      end
 
+      it 'does not mark the project as failed' do
         expect { worker.perform }.not_to change { project.reload.import_status }
       end
+    end
+  end
 
-      describe 'import without import_jid' do
-        it 'marks the project as failed' do
-          expect { worker.perform }.to change { project.reload.import_status }.to('failed')
+  describe 'with scheduled import_status' do
+    it_behaves_like 'project import job detection' do
+      let(:project) { create(:project, :import_scheduled, import_jid: '123') }
     end
   end
+
+  describe 'with started import_status' do
+    it_behaves_like 'project import job detection' do
+      let(:project) { create(:project, :import_started, import_jid: '123') }
     end
   end
 end