[ci skip] Merge branch 'master' into 43315-gpg-popover

* master: (39 commits)
  Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed
  Add changelog entry
  Use imported `DropdownUtils`
  Update CHANGELOG.md for 10.5.2
  Respond 404 when repo does not exist
  Fix deletion attempts on dropped tables
  Fix specs
  prevent localStorage.clear from running when it would cause an error
  Removed webpack tag
  move webpack bundles we intend to keep below the list of bundles we're refactoring to reduce conflicts
  Explicitly add uncovered files to troubleMakers
  Backport custom metrics ce components
  remove issue_show webpack bundle in favor of pages/projects/issues/show/index.js
  Revert "Revert "Use Dir.mktmpdir""
  Revert "Use Dir.mktmpdir"
  Use Dir.mktmpdir
  Refactored webpack bundle tag for deploy keys
  Change FileUtils.cp to FileUtils.copy
  Bring back the initial commit
  Add a button to deploy a runner to a Kubernetes cluster in the settings page
  ...

CHANGELOG.md

@@ -2,6 +2,27 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 10.5.2 (2018-02-25)
+
+### Fixed (7 changes)
+
+- Fix single digit value clipping for stacked progress bar. !17217
+- Fix issue with cache key being empty when variable used as the key. !17260
+- Enable Legacy Authorization by default on Cluster creations. !17302
+- Allow branch names to be named the same as the sha it points to.
+- Fix 500 error when loading an invalid upload URL.
+- Don't attempt to update user tracked fields if database is in read-only.
+- Prevent MR Widget error when no CI configured.
+
+### Performance (5 changes)
+
+- Improve query performance for snippets dashboard. !17088
+- Only check LFS integrity for first ref in a push to avoid timeout. !17098
+- Improve query performance of MembersFinder. !17190
+- Increase feature flag cache TTL to one hour.
+- Improve performance of searching for and autocompleting of users.
+
+
 ## 10.5.1 (2018-02-22)
 
 - No changes.

app/assets/javascripts/boards/boards_bundle.js → app/assets/javascripts/boards/index.js

@@ -24,7 +24,7 @@ import './components/new_list_dropdown';
 import './components/modal/index';
 import '../vue_shared/vue_resource_interceptor';
 
-$(() => {
+export default () => {
   const $boardApp = document.getElementById('board-app');
   const Store = gl.issueBoards.BoardsStore;
   const ModalStore = gl.issueBoards.ModalStore;
@@ -236,4 +236,4 @@ $(() => {
       </div>
     `,
   });
-});
+};

app/assets/javascripts/deploy_keys/index.js

 import Vue from 'vue';
 import deployKeysApp from './components/app.vue';
 
-document.addEventListener('DOMContentLoaded', () => new Vue({
+export default () => new Vue({
   el: document.getElementById('js-deploy-keys'),
   components: {
     deployKeysApp,
@@ -18,4 +18,4 @@ document.addEventListener('DOMContentLoaded', () => new Vue({
       },
     });
   },
-}));
+});

app/assets/javascripts/dispatcher.js

@@ -42,31 +42,34 @@ var Dispatcher;
         });
       });
 
-      switch (page) {
-        case 'projects:merge_requests:index':
-        case 'projects:issues:index':
-        case 'projects:issues:show':
-        case 'projects:issues:new':
-        case 'projects:issues:edit':
-        case 'projects:merge_requests:creations:new':
-        case 'projects:merge_requests:creations:diffs':
-        case 'projects:merge_requests:edit':
-        case 'projects:merge_requests:show':
-        case 'projects:commit:show':
-        case 'projects:activity':
-        case 'projects:commits:show':
-        case 'projects:show':
-        case 'groups:show':
-        case 'projects:tree:show':
-        case 'projects:find_file:show':
-        case 'projects:blob:show':
-        case 'projects:blame:show':
-        case 'projects:network:show':
-        case 'projects:artifacts:browse':
-        case 'projects:artifacts:file':
+      const shortcutHandlerPages = [
+        'projects:activity',
+        'projects:artifacts:browse',
+        'projects:artifacts:file',
+        'projects:blame:show',
+        'projects:blob:show',
+        'projects:commit:show',
+        'projects:commits:show',
+        'projects:find_file:show',
+        'projects:issues:edit',
+        'projects:issues:index',
+        'projects:issues:new',
+        'projects:issues:show',
+        'projects:merge_requests:creations:diffs',
+        'projects:merge_requests:creations:new',
+        'projects:merge_requests:edit',
+        'projects:merge_requests:index',
+        'projects:merge_requests:show',
+        'projects:network:show',
+        'projects:show',
+        'projects:tree:show',
+        'groups:show',
+      ];
+
+      if (shortcutHandlerPages.indexOf(page) !== -1) {
         shortcut_handler = true;
-          break;
       }
+
       switch (path[0]) {
         case 'admin':
           switch (path[1]) {

app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js → app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue

+<script>
 import eventHub from '../event_hub';
 import FilteredSearchTokenizer from '../filtered_search_tokenizer';
 
 export default {
   name: 'RecentSearchesDropdownContent',
-
   props: {
     items: {
       type: Array,
@@ -19,7 +19,6 @@ export default {
       required: true,
     },
   },
-
   computed: {
     processedItems() {
       return this.items.map((item) => {
@@ -42,7 +41,6 @@ export default {
       return this.items.length > 0;
     },
   },
-
   methods: {
     onItemActivated(text) {
       eventHub.$emit('recentSearchesItemSelected', text);
@@ -54,8 +52,9 @@ export default {
       eventHub.$emit('requestClearRecentSearches');
     },
   },
-
-  template: `
+};
+</script>
+<template>
   <div>
     <div
       v-if="!isLocalStorageAvailable"
@@ -65,16 +64,20 @@ export default {
     <ul v-else-if="hasItems">
       <li
         v-for="(item, index) in processedItems"
-          :key="index">
+        :key="`processed-items-${index}`"
+      >
         <button
           type="button"
           class="filtered-search-history-dropdown-item"
           @click="onItemActivated(item.text)">
           <span>
             <span
-                v-for="(token, tokenIndex) in item.tokens"
-                class="filtered-search-history-dropdown-token">
-                <span class="name">{{ token.prefix }}</span><span class="value">{{ token.suffix }}</span>
+              class="filtered-search-history-dropdown-token"
+              v-for="(token, index) in item.tokens"
+              :key="`dropdown-token-${index}`"
+            >
+              <span class="name">{{ token.prefix }}</span>
+              <span class="value">{{ token.suffix }}</span>
             </span>
           </span>
           <span class="filtered-search-history-dropdown-search-token">
@@ -98,5 +101,4 @@ export default {
       You don't have any recent searches
     </div>
   </div>
-  `,
-};
+</template>

app/assets/javascripts/filtered_search/recent_searches_root.js

 import Vue from 'vue';
-import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content';
+import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content.vue';
 import eventHub from './event_hub';
 
 class RecentSearchesRoot {
@@ -33,7 +33,7 @@ class RecentSearchesRoot {
     this.vm = new Vue({
       el: this.wrapperElement,
       components: {
-        'recent-searches-dropdown-content': RecentSearchesDropdownContent,
+        RecentSearchesDropdownContent,
       },
       data() { return state; },
       template: `

app/assets/javascripts/labels_select.js

@@ -213,7 +213,7 @@ export default class LabelsSelect {
             }
           }
           if (label.duplicate) {
-            color = gl.DropdownUtils.duplicateLabelColor(label.color);
+            color = DropdownUtils.duplicateLabelColor(label.color);
           }
           else {
             if (label.color != null) {

app/assets/javascripts/pages/projects/boards/index.js

 import UsersSelect from '~/users_select';
 import ShortcutsNavigation from '~/shortcuts_navigation';
+import initBoards from '~/boards';
 
 document.addEventListener('DOMContentLoaded', () => {
   new UsersSelect(); // eslint-disable-line no-new
   new ShortcutsNavigation(); // eslint-disable-line no-new
+  initBoards();
 });

app/assets/javascripts/pages/projects/issues/show/index.js

@@ -3,6 +3,7 @@ import Issue from '~/issue';
 import ShortcutsIssuable from '~/shortcuts_issuable';
 import ZenMode from '~/zen_mode';
 import '~/notes/index';
+import '~/issue_show/index';
 
 document.addEventListener('DOMContentLoaded', () => {
   new Issue(); // eslint-disable-line no-new

app/assets/javascripts/pages/projects/settings/repository/show/index.js

 import initSettingsPanels from '~/settings_panels';
+import initDeployKeys from '~/deploy_keys';
 
-document.addEventListener('DOMContentLoaded', initSettingsPanels);
+document.addEventListener('DOMContentLoaded', () => {
+  initDeployKeys();
+  initSettingsPanels();
+});

app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js

@@ -227,7 +227,8 @@ export default {
               @click="handleMergeButtonClick()"
               :disabled="isMergeButtonDisabled"
               :class="mergeButtonClass"
-              type="button">
+              type="button"
+              class="qa-merge-button">
               <i
                 v-if="isMakingRequest"
                 class="fa fa-spinner fa-spin"

app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue

@@ -111,7 +111,7 @@ js-toggle-container accept-action media space-children"
         >
           <button
             type="button"
-            class="btn btn-sm btn-reopen btn-success"
+            class="btn btn-sm btn-reopen btn-success qa-mr-rebase-button"
             :disabled="isMakingRequest"
             @click="rebase"
           >

app/controllers/projects/pages_domains_controller.rb

@@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController
 
   before_action :require_pages_enabled!
   before_action :authorize_update_pages!, except: [:show]
-  before_action :domain, only: [:show, :destroy]
+  before_action :domain, only: [:show, :destroy, :verify]
 
   def show
   end
@@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController
     @domain = @project.pages_domains.new
   end
 
+  def verify
+    result = VerifyPagesDomainService.new(@domain).execute
+
+    if result[:status] == :success
+      flash[:notice] = 'Successfully verified domain ownership'
+    else
+      flash[:alert] = 'Failed to verify domain ownership'
+    end
+
+    redirect_to project_pages_domain_path(@project, @domain)
+  end
+
   def create
     @domain = @project.pages_domains.create(pages_domain_params)
 
     if @domain.valid?
-      redirect_to project_pages_path(@project)
+      redirect_to project_pages_domain_path(@project, @domain)
     else
       render 'new'
     end
@@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController
   end
 
   def domain
-    @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s)
+    @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s)
   end
 end

app/controllers/projects/prometheus/metrics_controller.rb

+module Projects
+  module Prometheus
+    class MetricsController < Projects::ApplicationController
+      before_action :authorize_admin_project!
+
+      def active_common
+        respond_to do |format|
+          format.json do
+            matched_metrics = prometheus_service.matched_metrics || {}
+
+            if matched_metrics.any?
+              render json: matched_metrics
+            else
+              head :no_content
+            end
+          end
+        end
+      end
+
+      private
+
+      def prometheus_service
+        @prometheus_service ||= project.find_or_initialize_service('prometheus')
+      end
+    end
+  end
+end

app/controllers/projects/prometheus_controller.rb

-class Projects::PrometheusController < Projects::ApplicationController
-  before_action :authorize_read_project!
-  before_action :require_prometheus_metrics!
-
-  def active_metrics
-    respond_to do |format|
-      format.json do
-        matched_metrics = project.prometheus_service.matched_metrics || {}
-
-        if matched_metrics.any?
-          render json: matched_metrics
-        else
-          head :no_content
-        end
-      end
-    end
-  end
-
-  private
-
-  def require_prometheus_metrics!
-    render_404 unless project.prometheus_service.present?
-  end
-end

app/helpers/application_settings_helper.rb

@@ -199,6 +199,7 @@ module ApplicationSettingsHelper
       :metrics_port,
       :metrics_sample_interval,
       :metrics_timeout,
+      :pages_domain_verification_enabled,
       :password_authentication_enabled_for_web,
       :password_authentication_enabled_for_git,
       :performance_bar_allowed_group_id,

app/mailers/emails/pages_domains.rb

+module Emails
+  module PagesDomains
+    def pages_domain_enabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled")
+      )
+    end
+
+    def pages_domain_disabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled")
+      )
+    end
+
+    def pages_domain_verification_succeeded_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+
+    def pages_domain_verification_failed_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+  end
+end

app/mailers/notify.rb

@@ -5,6 +5,7 @@ class Notify < BaseMailer
   include Emails::Issues
   include Emails::MergeRequests
   include Emails::Notes
+  include Emails::PagesDomains
   include Emails::Projects
   include Emails::Profile
   include Emails::Pipelines

app/models/pages_domain.rb

 class PagesDomain < ActiveRecord::Base
+  VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze
+  VERIFICATION_THRESHOLD = 3.days.freeze
+
   belongs_to :project
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }
   validates :certificate, certificate: true, allow_nil: true, allow_blank: true
   validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+  validates :verification_code, presence: true, allow_blank: false
 
   validate :validate_pages_domain
   validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? }
@@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
+  after_initialize :set_verification_code
   after_create :update_daemon
-  after_save :update_daemon
+  after_update :update_daemon, if: :pages_config_changed?
   after_destroy :update_daemon
 
+  scope :enabled, -> { where('enabled_until >= ?', Time.now ) }
+  scope :needs_verification, -> do
+    verified_at = arel_table[:verified_at]
+    enabled_until = arel_table[:enabled_until]
+    threshold = Time.now + VERIFICATION_THRESHOLD
+
+    where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold))))
+  end
+
+  def verified?
+    !!verified_at
+  end
+
+  def unverified?
+    !verified?
+  end
+
+  def enabled?
+    !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present?
+  end
+
   def to_param
     domain
   end
@@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base
     @certificate_text ||= x509.try(:to_text)
   end
 
+  # Verification codes may be TXT records for domain or verification_domain, to
+  # support the use of CNAME records on domain.
+  def verification_domain
+    return unless domain.present?
+
+    "_#{VERIFICATION_KEY}.#{domain}"
+  end
+
+  def keyed_verification_code
+    return unless verification_code.present?
+
+    "#{VERIFICATION_KEY}=#{verification_code}"
+  end
+
   private
 
+  def set_verification_code
+    return if self.verification_code.present?
+
+    self.verification_code = SecureRandom.hex(16)
+  end
+
   def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
+  def pages_config_changed?
+    project_id_changed? ||
+      domain_changed? ||
+      certificate_changed? ||
+      key_changed? ||
+      became_enabled? ||
+      became_disabled?
+  end
+
+  def became_enabled?
+    enabled_until.present? && !enabled_until_was.present?
+  end
+
+  def became_disabled?
+    !enabled_until.present? && enabled_until_was.present?
+  end
+
   def validate_matching_key
     unless has_matching_key?
       self.errors.add(:key, "doesn't match the certificate")

app/models/project_services/prometheus_service.rb

@@ -69,16 +69,16 @@ class PrometheusService < MonitoringService
     client.ping
 
     { success: true, result: 'Checked API endpoint' }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err }
   end
 
   def environment_metrics(environment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
+    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics))
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics))
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics))
     metrics&.merge(deployment_time: deployment.created_at.to_i) || {}
   end
 
@@ -107,7 +107,7 @@ class PrometheusService < MonitoringService
       data: data,
       last_update: Time.now.utc
     }
-  rescue Gitlab::PrometheusError => err
+  rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err.message }
   end
 
@@ -116,10 +116,10 @@ class PrometheusService < MonitoringService
       Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url))
     else
       cluster = cluster_with_prometheus(environment_id)
-      raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster
+      raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster
 
       rest_client = client_from_cluster(cluster)
-      raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client
+      raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client
 
       Gitlab::PrometheusClient.new(rest_client)
     end
@@ -152,10 +152,12 @@ class PrometheusService < MonitoringService
     cluster.application_prometheus.proxy_client
   end
 
-  def rename_data_to_metrics(metrics)
-    metrics[:metrics] = metrics.delete :data
+  def rename_field(old_field, new_field)
+    -> (metrics) do
+      metrics[new_field] = metrics.delete(old_field)
       metrics
     end
+  end
 
   def synchronize_service_state!
     self.active = prometheus_installed? || manual_configuration?

app/services/ci/create_trace_artifact_service.rb

@@ -4,13 +4,33 @@ module Ci
       return if job.job_artifacts_trace
 
       job.trace.read do |stream|
-        if stream.file?
+        break unless stream.file?
+
+        clone_file!(stream.path, JobArtifactUploader.workhorse_upload_path) do |clone_path|
+          create_job_trace!(job, clone_path)
+          FileUtils.rm(stream.path)
+        end
+      end
+    end
+
+    private
+
+    def create_job_trace!(job, path)
+      File.open(path) do |stream|
         job.create_job_artifacts_trace!(
           project: job.project,
           file_type: :trace,
           file: stream)
       end
     end
+
+    def clone_file!(src_path, temp_dir)
+      FileUtils.mkdir_p(temp_dir)
+      Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path|
+        temp_path = File.join(dir_path, "job.log")
+        FileUtils.copy(src_path, temp_path)
+        yield(temp_path)
+      end
     end
   end
 end

app/services/notification_service.rb

@@ -339,6 +339,30 @@ class NotificationService
     end
   end
 
+  def pages_domain_verification_succeeded(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_verification_failed(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_failed_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_enabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_enabled_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_disabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_disabled_email(domain, user).deliver_later
+    end
+  end
+
   protected
 
   def new_resource_email(target, method)
@@ -433,6 +457,14 @@ class NotificationService
 
   private
 
+  def recipients_for_pages_domain(domain)
+    project = domain.project
+
+    return [] unless project
+
+    notifiable_users(project.team.masters, :watch, target: project)
+  end
+
   def notifiable?(*args)
     NotificationRecipientService.notifiable?(*args)
   end

app/services/projects/update_pages_configuration_service.rb

@@ -23,7 +23,7 @@ module Projects
     end
 
     def pages_domains_config
-      project.pages_domains.map do |domain|
+      enabled_pages_domains.map do |domain|
         {
           domain: domain.domain,
           certificate: domain.certificate,
@@ -32,6 +32,14 @@ module Projects
       end
     end
 
+    def enabled_pages_domains
+      if Gitlab::CurrentSettings.pages_domain_verification_enabled?
+        project.pages_domains.enabled
+      else
+        project.pages_domains
+      end
+    end
+
     def reload_daemon
       # GitLab Pages daemon constantly watches for modification time of `pages.path`
       # It reloads configuration when `pages.path` is modified

app/services/verify_pages_domain_service.rb

+require 'resolv'
+
+class VerifyPagesDomainService < BaseService
+  # The maximum number of seconds to be spent on each DNS lookup
+  RESOLVER_TIMEOUT_SECONDS = 15
+
+  # How long verification lasts for
+  VERIFICATION_PERIOD = 7.days
+
+  attr_reader :domain
+
+  def initialize(domain)
+    @domain = domain
+  end
+
+  def execute
+    return error("No verification code set for #{domain.domain}") unless domain.verification_code.present?
+
+    if !verification_enabled? || dns_record_present?
+      verify_domain!
+    elsif expired?
+      disable_domain!
+    else
+      unverify_domain!
+    end
+  end
+
+  private
+
+  def verify_domain!
+    was_disabled = !domain.enabled?
+    was_unverified = domain.unverified?
+
+    # Prevent any pre-existing grace period from being truncated
+    reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max
+
+    domain.update!(verified_at: Time.now, enabled_until: reverify)
+
+    if was_disabled
+      notify(:enabled)
+    elsif was_unverified
+      notify(:verification_succeeded)
+    end
+
+    success
+  end
+
+  def unverify_domain!
+    if domain.verified?
+      domain.update!(verified_at: nil)
+      notify(:verification_failed)
+    end
+
+    error("Couldn't verify #{domain.domain}")
+  end
+
+  def disable_domain!
+    domain.update!(verified_at: nil, enabled_until: nil)
+
+    notify(:disabled)
+
+    error("Couldn't verify #{domain.domain}. It is now disabled.")
+  end
+
+  # A domain is only expired until `disable!` has been called
+  def expired?
+    domain.enabled_until && domain.enabled_until < Time.now
+  end
+
+  def dns_record_present?
+    Resolv::DNS.open do |resolver|
+      resolver.timeouts = RESOLVER_TIMEOUT_SECONDS
+
+      check(domain.domain, resolver) || check(domain.verification_domain, resolver)
+    end
+  end
+
+  def check(domain_name, resolver)
+    records = parse(txt_records(domain_name, resolver))
+
+    records.any? do |record|
+      record == domain.keyed_verification_code || record == domain.verification_code
+    end
+  rescue => err
+    log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}")
+    false
+  end
+
+  def txt_records(domain_name, resolver)
+    resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT)
+  end
+
+  def parse(records)
+    records.flat_map(&:strings).flat_map(&:split)
+  end
+
+  def verification_enabled?
+    Gitlab::CurrentSettings.pages_domain_verification_enabled?
+  end
+
+  def notify(type)
+    return unless verification_enabled?
+
+    Gitlab::AppLogger.info("Pages domain '#{domain.domain}' changed state to '#{type}'")
+    notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -237,6 +237,17 @@
       .col-sm-10
         = f.number_field :max_pages_size, class: 'form-control'
         .help-block 0 for unlimited
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :pages_domain_verification_enabled do
+            = f.check_box :pages_domain_verification_enabled
+            Require users to prove ownership of custom domains
+        .help-block
+          Domain verification is an essential security measure for public GitLab
+          sites. Users are required to demonstrate they control a domain before
+          it is enabled
+          = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
 
   %fieldset
     %legend Continuous Integration and Deployment

app/views/admin/runners/index.html.haml

@@ -35,9 +35,8 @@
           method: :put, class: 'btn btn-default',
           data: { confirm: _("Are you sure you want to reset registration token?") }
 
-  = render partial: 'ci/runner/how_to_setup_runner',
-           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token,
-                     type: 'shared' }
+  = render partial: 'ci/runner/how_to_setup_shared_runner',
+           locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token }
 
   .append-bottom-20.clearfix
     .pull-left

app/views/ci/runner/_how_to_setup_runner.html.haml

 - link = link_to _("GitLab Runner section"), 'https://about.gitlab.com/gitlab-ci/#gitlab-runner', target: '_blank'
-.bs-callout.help-callout
-  %h4= _("How to setup a #{type} Runner for a new project")
+.append-bottom-10
+  %h4= _("Setup a #{type} Runner manually")
 
-  %ol
+%ol
   %li
     = _("Install a Runner compatible with GitLab CI")
     = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe

app/views/ci/runner/_how_to_setup_shared_runner.html.haml

+.bs-callout.help-callout
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'shared' }

app/views/ci/runner/_how_to_setup_specific_runner.html.haml

+.bs-callout.help-callout
+  .append-bottom-10
+    %h4= _('Setup a specific Runner automatically')
+
+  %p
+    - link_to_help_page = link_to(_('Learn more about Kubernetes'),
+                                  help_page_path('user/project/clusters/index'),
+                                  target: '_blank',
+                                  rel: 'noopener noreferrer')
+
+    = _('You can easily install a Runner on a Kubernetes cluster. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page }
+
+  %ol
+    %li
+      = _('Click the button below to begin the install process by navigating to the Kubernetes page')
+    %li
+      = _('Select an existing Kubernetes cluster or create a new one')
+    %li
+      = _('From the Kubernetes cluster details view, install Runner from the applications list')
+
+  = link_to _('Install Runner on Kubernetes'),
+    project_clusters_path(@project),
+    class: 'btn btn-info'
+  %hr
+  = render partial: 'ci/runner/how_to_setup_runner',
+           locals: { registration_token: registration_token, type: 'specific' }

app/views/notify/pages_domain_disabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  %strong disabled.
+  This means that your content is no longer visible at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  If this domain has been disabled in error, please follow
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  to verify and re-enable your domain.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_disabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+**disabled**. This means that your content is no longer visible at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+If this domain has been disabled in error, please follow these instructions
+to verify and re-enable your domain:
+
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_enabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_enabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+enabled. You should now be able to view your content at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/notify/pages_domain_verification_failed_email.html.haml

+%p
+  Verification has failed for one of your GitLab Pages custom domains!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Unless you take action, it will be disabled on
+  %strong= @domain.enabled_until.strftime('%F %T.')
+  Until then, you can view your content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_failed_email.text.haml

+Verification has failed for one of your GitLab Pages custom domains!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*.
+Until then, you can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_succeeded_email.html.haml

+%p
+  One of your GitLab Pages custom domains has been successfully verified!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  This is a notification. No action is required on your part. You can view your
+  content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_verification_succeeded_email.text.haml

+One of your GitLab Pages custom domains has been successfully verified!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+No action is required on your part. You can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/projects/_merge_request_fast_forward_settings.html.haml

@@ -3,7 +3,7 @@
 
 .radio
   = label_tag :project_merge_method_ff do
-    = form.radio_button :merge_method, :ff, class: "js-merge-method-radio"
+    = form.radio_button :merge_method, :ff, class: "js-merge-method-radio qa-radio-button-merge-ff"
     %strong Fast-forward merge
     %br
     %span.descr

app/views/projects/clusters/_empty_state.html.haml

@@ -4,7 +4,7 @@
   .col-xs-12
     .text-content
       %h4.text-center= s_('ClusterIntegration|Integrate Kubernetes cluster automation')
-      - link_to_help_page = link_to(s_('ClusterIntegration|Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
+      - link_to_help_page = link_to(_('Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer')
       %p= s_('ClusterIntegration|Kubernetes clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page}
 
       .text-center

app/views/projects/edit.html.haml

@@ -85,7 +85,7 @@
     .settings-content
       = form_for [@project.namespace.becomes(Namespace), @project], remote: true, html: { multipart: true, class: "merge-request-settings-form" }, authenticity_token: true do |f|
         = render 'merge_request_settings', form: f
-        = f.submit 'Save changes', class: "btn btn-save"
+        = f.submit 'Save changes', class: "btn btn-save qa-save-merge-request-changes"
 
   = render 'export', project: @project
 

app/views/projects/issues/show.html.haml

@@ -82,6 +82,3 @@
     = render 'projects/issues/discussion'
 
 = render 'shared/issuable/sidebar', issuable: @issue
-
-= webpack_bundle_tag('common_vue')
-= webpack_bundle_tag('issue_show')

app/views/projects/pages/_list.html.haml

@@ -3,15 +3,26 @@
     .panel-heading
       Domains (#{@domains.count})
     %ul.well-list
+      - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
       - @domains.each do |domain|
         %li
           .pull-right
             = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped"
             = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped"
           .clearfix
-            %span= link_to domain.domain, domain.url
+            - if verification_enabled
+              - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success']
+              = link_to domain.url, title: tooltip, class: 'has-tooltip' do
+                = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}")
+                = domain.domain
+            - else
+              = link_to domain.domain, domain.url
           %p
             - if domain.subject
               %span.label.label-gray Certificate: #{domain.subject}
             - if domain.expired?
               %span.label.label-danger Expired
+        - if verification_enabled && domain.unverified?
+          %li.warning-row
+            #{domain.domain} is not verified. To learn how to verify ownership, visit your
+            = link_to 'domain details', project_pages_domain_path(@project, domain)

app/views/projects/pages_domains/show.html.haml

 - page_title "#{@domain.domain}", 'Pages Domains'
+- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
+- if verification_enabled && @domain.unverified?
+  %p.alert.alert-warning
+    %strong
+      This domain is not verified. You will need to verify ownership before
+      access is enabled.
 
 %h3.page-title
   Pages Domain
@@ -15,9 +21,26 @@
         DNS
       %td
         %p
-          To access the domain create a new DNS record:
+          To access this domain create a new DNS record:
         %pre
           #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}.
+    - if verification_enabled
+      %tr
+        %td
+          Verification status
+        %td
+          %p
+            - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+            To #{link_to 'verify ownership', help_link} of your domain, create
+            this DNS record:
+          %pre
+            #{@domain.verification_domain} TXT #{@domain.keyed_verification_code}
+          %p
+            - if @domain.verified?
+              #{@domain.domain} has been successfully verified.
+            - else
+              = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm'
+
     %tr
       %td
         Certificate

app/views/projects/runners/_shared_runners.html.haml

 %h3 Shared Runners
 
-.bs-callout.bs-callout-warning.shared-runners-description
+.bs-callout.shared-runners-description
   - if Gitlab::CurrentSettings.shared_runners_text.present?
     = markdown_field(Gitlab::CurrentSettings.current_application_settings, :shared_runners_text)
   - else
@@ -9,7 +9,7 @@
     on GitLab.com).
   %hr
   - if @project.shared_runners_enabled?
-    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do
+    = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do
       Disable shared Runners
   - else
     = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do

app/views/projects/runners/_specific_runners.html.haml

 %h3 Specific Runners
 
-= render partial: 'ci/runner/how_to_setup_runner',
-         locals: { registration_token: @project.runners_token,
-                   type: 'specific' }
+= render partial: 'ci/runner/how_to_setup_specific_runner',
+         locals: { registration_token: @project.runners_token }
 
 - if @project_runners.any?
   %h4.underlined-title Runners activated for this project

app/views/projects/services/prometheus/_show.html.haml

@@ -7,7 +7,7 @@
       = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus')
 
   .col-lg-9
-    .panel.panel-default.js-panel-monitored-metrics{ data: { "active-metrics" => "#{project_prometheus_active_metrics_path(@project, :json)}" } }
+    .panel.panel-default.js-panel-monitored-metrics{ data: { active_metrics: active_common_project_prometheus_metrics_path(@project, :json) } }
       .panel-heading
         %h3.panel-title
           = s_('PrometheusService|Monitored')

app/views/projects/settings/repository/show.html.haml

@@ -4,7 +4,6 @@
 
 - content_for :page_specific_javascripts do
   = webpack_bundle_tag('common_vue')
-  = webpack_bundle_tag('deploy_keys')
 
 -# Protected branches & tags use a lot of nested partials.
 -# The shared parts of the views can be found in the `shared` directory.

app/views/shared/boards/_show.html.haml

@@ -6,7 +6,6 @@
 
 - content_for :page_specific_javascripts do
   = webpack_bundle_tag 'common_vue'
-  = webpack_bundle_tag 'boards'
 
   %script#js-board-template{ type: "text/x-template" }= render "shared/boards/components/board"
   %script#js-board-modal-filter{ type: "text/x-template" }= render "shared/issuable/search_bar", type: :boards_modal

app/workers/all_queues.yml

@@ -3,6 +3,7 @@
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup
+- cronjob:pages_domain_verification_cron
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
 - cronjob:remove_expired_group_links
@@ -82,6 +83,7 @@
 - new_merge_request
 - new_note
 - pages
+- pages_domain_verification
 - post_receive
 - process_commit
 - project_cache

app/workers/pages_domain_verification_cron_worker.rb

+class PagesDomainVerificationCronWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    PagesDomain.needs_verification.find_each do |domain|
+      PagesDomainVerificationWorker.perform_async(domain.id)
+    end
+  end
+end

app/workers/pages_domain_verification_worker.rb

+class PagesDomainVerificationWorker
+  include ApplicationWorker
+
+  def perform(domain_id)
+    domain = PagesDomain.find_by(id: domain_id)
+
+    return unless domain
+
+    VerifyPagesDomainService.new(domain).execute
+  end
+end

app/workers/stuck_import_jobs_worker.rb

@@ -16,43 +16,41 @@ class StuckImportJobsWorker
   private
 
   def mark_projects_without_jid_as_failed!
-    started_projects_without_jid.each do |project|
+    enqueued_projects_without_jid.each do |project|
       project.mark_import_as_failed(error_message)
     end.count
   end
 
   def mark_projects_with_jid_as_failed!
-    completed_jids_count = 0
-
-    started_projects_with_jid.find_in_batches(batch_size: 500) do |group|
-      jids = group.map(&:import_jid)
+    jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h
 
     # Find the jobs that aren't currently running or that exceeded the threshold.
-      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set
+    completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
+    return unless completed_jids.any?
 
-      if completed_jids.any?
-        completed_jids_count += completed_jids.count
-        group.each do |project|
-          project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid)
-        end
+    completed_project_ids = jids_and_ids.values_at(*completed_jids)
 
-        Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}")
-      end
-    end
+    # We select the projects again, because they may have transitioned from
+    # scheduled/started to finished/failed while we were looking up their Sidekiq status.
+    completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids)
 
-    completed_jids_count
+    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}")
+
+    completed_projects.each do |project|
+      project.mark_import_as_failed(error_message)
+    end.count
   end
 
-  def started_projects
-    Project.with_import_status(:started)
+  def enqueued_projects
+    Project.with_import_status(:scheduled, :started)
   end
 
-  def started_projects_with_jid
-    started_projects.where.not(import_jid: nil)
+  def enqueued_projects_with_jid
+    enqueued_projects.where.not(import_jid: nil)
   end
 
-  def started_projects_without_jid
-    started_projects.where(import_jid: nil)
+  def enqueued_projects_without_jid
+    enqueued_projects.where(import_jid: nil)
   end
 
   def error_message

changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml

+---
+title: Add verification for GitLab Pages custom domains
+merge_request:
+author:
+type: security

changelogs/unreleased/34130-null-pipes.yml

----
-title: Prevent MR Widget error when no CI configured
-merge_request:
-author:
-type: fixed

changelogs/unreleased/41461-project-members-slow-due-to-sql.yml

----
-title: Improve query performance of MembersFinder.
-merge_request: 17190
-author:
-type: performance

changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml

----
-title: Enable Legacy Authorization by default on Cluster creations
-merge_request: 17302
-author:
-type: fixed

changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml

+---
+title: Add a button to deploy a runner to a Kubernetes cluster in the settings page
+merge_request: 17278
+author:
+type: changed

changelogs/unreleased/42877-snippets-dashboard-slow.yml

----
-title: Improve query performance for snippets dashboard.
-merge_request: 17088
-author:
-type: performance

changelogs/unreleased/43373-fix-cache-index-appending.yml

----
-title: Fix issue with cache key being empty when variable used as the key
-merge_request: 17260
-author:
-type: fixed

changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml

+---
+title: Fix Group labels load failure when there are duplicate labels present
+merge_request: 17353
+author:
+type: fixed

changelogs/unreleased/fix-500-for-invalid-upload-path.yml → changelogs/unreleased/dm-stuck-import-jobs-verify.yml

 ---
-title: Fix 500 error when loading an invalid upload URL
+title: Verify project import status again before marking as failed
 merge_request:
 author:
 type: fixed

changelogs/unreleased/flipper-caching.yml

----
-title: Increase feature flag cache TTL to one hour
-merge_request:
-author:
-type: performance

changelogs/unreleased/grpc-unavailable-restart.yml

+---
+title: Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed
+merge_request: 17293
+author:
+type: fixed

changelogs/unreleased/jej-fix-slow-lfs-object-check.yml

----
-title: Only check LFS integrity for first ref in a push to avoid timeout
-merge_request: 17098
-author:
-type: performance

changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml

----
-title: Fix single digit value clipping for stacked progress bar
-merge_request: 17217
-author:
-type: fixed

changelogs/unreleased/minimal-fix-for-artifacts-service.yml

+---
+title: Prevent trace artifact migration to incur data loss
+merge_request: 17313
+author:
+type: fixed

changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml

+---
+title: Return a 404 instead of 403 if the repository does not exist on disk
+merge_request: 17341
+author:
+type: fixed

changelogs/unreleased/refactor-move-filtered-search-vue-component.yml

+---
+title: Move RecentSearchesDropdownContent vue component
+merge_request: 16951
+author: George Tsiolis
+type: performance

changelogs/unreleased/sh-guard-read-only-user-updates.yml

----
-title: Don't attempt to update user tracked fields if database is in read-only
-merge_request:
-author:
-type: fixed

changelogs/unreleased/users-autocomplete.yml

----
-title: Improve performance of searching for and autocompleting of users
-merge_request:
-author:
-type: performance

changelogs/unreleased/zj-branch-contains-git-message.yml

----
-title: Allow branch names to be named the same as the sha it points to
-merge_request:
-author:
-type: fixed

config/gitlab.yml.example

@@ -214,6 +214,10 @@ production: &base
     repository_archive_cache_worker:
       cron: "0 * * * *"
 
+    # Verify custom GitLab Pages domains
+    pages_domain_verification_cron_worker:
+      cron: "*/15 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
 
+Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *'
+Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker'
+
 #
 # GitLab Shell
 #

config/initializers/sidekiq.rb

@@ -10,7 +10,7 @@ Sidekiq.configure_server do |config|
 
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
-    chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
+    chain.add Gitlab::SidekiqMiddleware::Shutdown
     chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
     chain.add Gitlab::SidekiqStatus::ServerMiddleware
   end

config/routes/project.rb

@@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do
       end
 
       resource :pages, only: [:show, :destroy] do
-        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} }
+        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do
+          member do
+            post :verify
+          end
+        end
       end
 
       resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do
@@ -74,7 +78,9 @@ constraints(ProjectUrlConstrainer.new) do
       resource :mattermost, only: [:new, :create]
 
       namespace :prometheus do
-        get :active_metrics
+        resources :metrics, constraints: { id: %r{[^\/]+} }, only: [] do
+          get :active_common, on: :collection
+        end
       end
 
       resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do

config/sidekiq_queues.yml

@@ -67,3 +67,4 @@
   - [gcp_cluster, 1]
   - [project_migrate_hashed_storage, 1]
   - [storage_migrator, 1]
+  - [pages_domain_verification, 1]

config/webpack.config.js

@@ -52,20 +52,15 @@ var config = {
   entry: {
     balsamiq_viewer:      './blob/balsamiq_viewer.js',
     blob:                 './blob_edit/blob_bundle.js',
-    boards:               './boards/boards_bundle.js',
     common:               './commons/index.js',
     common_vue:           './vue_shared/vue_resource_interceptor.js',
     cycle_analytics:      './cycle_analytics/cycle_analytics_bundle.js',
     commit_pipelines:     './commit/pipelines/pipelines_bundle.js',
-    deploy_keys:          './deploy_keys/index.js',
     diff_notes:           './diff_notes/diff_notes_bundle.js',
     environments:         './environments/environments_bundle.js',
     environments_folder:  './environments/folder/environments_folder_bundle.js',
     filtered_search:      './filtered_search/filtered_search_bundle.js',
     help:                 './help/help.js',
-    issue_show:           './issue_show/index.js',
-    locale:               './locale/index.js',
-    main:                 './main.js',
     merge_conflicts:      './merge_conflicts/merge_conflicts_bundle.js',
     monitoring:           './monitoring/monitoring_bundle.js',
     network:              './network/network_bundle.js',
@@ -78,18 +73,24 @@ var config = {
     protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
     registry_list:        './registry/index.js',
-    ide:                  './ide/index.js',
     sidebar:              './sidebar/sidebar_bundle.js',
     snippet:              './snippet/snippet_bundle.js',
     sketch_viewer:        './blob/sketch_viewer.js',
     stl_viewer:           './blob/stl_viewer.js',
     terminal:             './terminal/terminal_bundle.js',
-    u2f:                  ['vendor/u2f'],
     ui_development_kit:   './ui_development_kit.js',
-    raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
-    test:                 './test.js',
     two_factor_auth:      './two_factor_auth.js',
+
+
+    common:               './commons/index.js',
+    common_vue:           './vue_shared/vue_resource_interceptor.js',
+    locale:               './locale/index.js',
+    main:                 './main.js',
+    ide:                  './ide/index.js',
+    raven:                './raven/index.js',
+    test:                 './test.js',
+    u2f:                  ['vendor/u2f'],
     webpack_runtime:      './webpack.js',
   },
 
@@ -252,7 +253,6 @@ var config = {
         'environments_folder',
         'filtered_search',
         'groups',
-        'issue_show',
         'merge_conflicts',
         'monitoring',
         'notebook_viewer',

db/migrate/20180216120000_add_pages_domain_verification.rb

+class AddPagesDomainVerification < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :verified_at, :datetime_with_timezone
+    add_column :pages_domains, :verification_code, :string
+  end
+end

db/migrate/20180216120010_add_pages_domain_verified_at_index.rb

+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, :verified_at
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, :verified_at
+  end
+end

db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb

+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+  end
+end

db/migrate/20180216120030_add_pages_domain_enabled_until.rb

+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :enabled_until, :datetime_with_timezone
+  end
+end

db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb

+class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, [:project_id, :enabled_until]
+    add_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+    remove_concurrent_index :pages_domains, [:project_id, :enabled_until]
+  end
+end

db/migrate/20180216120050_pages_domains_verification_grace_period.rb

+class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    now = Time.now
+    grace = now + 30.days
+
+    PagesDomain.each_batch do |relation|
+      relation.update_all(verified_at: now, enabled_until: grace)
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb

+class FillPagesDomainVerificationCode < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    PagesDomain.where(verification_code: [nil, '']).each_batch do |relation|
+      connection.execute(set_codes_sql(relation))
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+
+    change_column_null(:pages_domains, :verification_code, false)
+  end
+
+  def down
+    change_column_null(:pages_domains, :verification_code, true)
+  end
+
+  private
+
+  def set_codes_sql(relation)
+    ids = relation.pluck(:id)
+    whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" }
+
+    <<~SQL
+      UPDATE pages_domains
+      SET verification_code =
+        CASE id
+        #{whens.join("\n")}
+        END
+      WHERE id IN(#{ids.join(',')})
+    SQL
+  end
+end

db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb

+class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    PagesDomain.each_batch do |relation|
+      ids = relation.pluck(:id).map { |id| [id] }
+      PagesDomainVerificationWorker.bulk_perform_async(ids)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180215181245) do
+ActiveRecord::Schema.define(version: 20180216121030) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.integer "gitaly_timeout_fast", default: 10, null: false
     t.boolean "authorized_keys_enabled", default: true, null: false
     t.string "auto_devops_domain"
+    t.boolean "pages_domain_verification_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -1313,10 +1314,16 @@ ActiveRecord::Schema.define(version: 20180215181245) do
     t.string "encrypted_key_iv"
     t.string "encrypted_key_salt"
     t.string "domain"
+    t.datetime_with_timezone "verified_at"
+    t.string "verification_code", null: false
+    t.datetime_with_timezone "enabled_until"
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
   add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
+  add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree
+  add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false

doc/administration/pages/index.md

@@ -226,6 +226,18 @@ world. Custom domains and TLS are supported.
 
 1. [Reconfigure GitLab][reconfigure]
 
+### Custom domain verification
+
+To prevent malicious users from hijacking domains that don't belong to them,
+GitLab supports [custom domain verification](../../user/project/pages/getting_started_part_three.md#dns-txt-record).
+When adding a custom domain, users will be required to prove they own it by
+adding a GitLab-controlled verification code to the DNS records for that domain.
+
+If your userbase is private or otherwise trusted, you can disable the
+verification requirement. Navigate to `Admin area ➔ Settings` and uncheck
+**Require users to prove ownership of custom domains** in the Pages section.
+This setting is enabled by default.
+
 ## Change storage path
 
 Follow the steps below to change the default path where GitLab Pages' contents

doc/user/project/pages/getting_started_part_three.md

@@ -62,7 +62,7 @@ for the most popular hosting services:
 - [Microsoft](https://msdn.microsoft.com/en-us/library/bb727018.aspx)
 
 If your hosting service is not listed above, you can just try to
-search the web for "how to add dns record on <my hosting service>".
+search the web for `how to add dns record on <my hosting service>`.
 
 ### DNS A record
 
@@ -95,12 +95,32 @@ without any `/project-name`.
 
 ![DNS CNAME record pointing to GitLab.com project](img/dns_cname_record_example.png)
 
-### TL;DR
+#### DNS TXT record
+
+Unless your GitLab administrator has [disabled custom domain verification](../../../administration/pages/index.md#custom-domain-verification),
+you'll have to prove that you own the domain by creating a `TXT` record
+containing a verification code. The code will be displayed after you
+[add your custom domain to GitLab Pages settings](#add-your-custom-domain-to-gitlab-pages-settings).
+
+If using a [DNS A record](#dns-a-record), you can place the TXT record directly
+under the domain. If using a [DNS CNAME record](#dns-cname-record), the two record types won't
+co-exist, so you need to place the TXT record in a special subdomain of its own.
+
+#### TL;DR
+
+If the domain has multiple uses (e.g., you host email on it as well):
 
 | From | DNS Record | To |
 | ---- | ---------- | -- |
 | domain.com | A | 52.167.214.135 |
-| subdomain.domain.com | CNAME | namespace.gitlab.io |
+| domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
+
+If the domain is dedicated to GitLab Pages use and no other services run on it:
+
+| From | DNS Record | To |
+| ---- | ---------- | -- |
+| subdomain.domain.com | CNAME | gitlab.io |
+| _gitlab-pages-verification-code.subdomain.domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff |
 
 > **Notes**:
 >
@@ -121,6 +141,17 @@ your site will be accessible only via HTTP:
 
 ![Add new domain](img/add_certificate_to_pages.png)
 
+Once you have added a new domain, you will need to **verify your ownership**
+(unless the GitLab administrator has disabled this feature). A verification code
+will be shown to you; add it as a [DNS TXT record](#dns-txt-record), then press
+the "Verify ownership" button to activate your new domain:
+
+![Verify your domain](img/verify_your_domain.png)
+
+Once your domain has been verified, leave the verification record in place -
+your domain will be periodically reverified, and may be disabled if the record
+is removed.
+
 You can add more than one alias (custom domains and subdomains) to the same project.
 An alias can be understood as having many doors leading to the same room.
 
@@ -128,8 +159,8 @@ All the aliases you've set to your site will be listed on **Setting > Pages**.
 From that page, you can view, add, and remove them.
 
 Note that [DNS propagation may take some time (up to 24h)](http://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/domain-names-dns-changes),
-although it's usually a matter of minutes to complete. Until it does, visit attempts
-to your domain will respond with a 404.
+although it's usually a matter of minutes to complete. Until it does, verification
+will fail and attempts to visit your domain will respond with a 404.
 
 Read through the [general documentation on GitLab Pages](introduction.md#add-a-custom-domain-to-your-pages-website) to learn more about adding
 custom domains to GitLab Pages sites.

lib/api/entities.rb

@@ -1154,6 +1154,10 @@ module API
       expose :domain
       expose :url
       expose :project_id
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         as: :certificate_expiration,
         if: ->(pages_domain, _) { pages_domain.certificate? },
@@ -1165,6 +1169,10 @@ module API
     class PagesDomain < Grape::Entity
       expose :domain
       expose :url
+      expose :verified?, as: :verified
+      expose :verification_code, as: :verification_code
+      expose :enabled_until
+
       expose :certificate,
         if: ->(pages_domain, _) { pages_domain.certificate? },
         using: PagesDomainCertificate do |pages_domain|

lib/gitlab/git_access.rb

@@ -199,7 +199,7 @@ module Gitlab
 
     def check_repository_existence!
       unless repository.exists?
-        raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
+        raise NotFoundError, ERROR_MESSAGES[:no_repo]
       end
     end
 

lib/gitlab/gitaly_client.rb

@@ -125,6 +125,8 @@ module Gitlab
       kwargs = yield(kwargs) if block_given?
 
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
+    rescue GRPC::Unavailable => ex
+      handle_grpc_unavailable!(ex)
     ensure
       duration = Gitlab::Metrics::System.monotonic_time - start
 
@@ -135,6 +137,27 @@ module Gitlab
         duration)
     end
 
+    def self.handle_grpc_unavailable!(ex)
+      status = ex.to_status
+      raise ex unless status.details == 'Endpoint read failed'
+
+      # There is a bug in grpc 1.8.x that causes a client process to get stuck
+      # always raising '14:Endpoint read failed'. The only thing that we can
+      # do to recover is to restart the process.
+      #
+      # See https://gitlab.com/gitlab-org/gitaly/issues/1029
+
+      if Sidekiq.server?
+        raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s)
+      else
+        # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request.
+        Process.kill('QUIT', Process.pid)
+      end
+
+      raise ex
+    end
+    private_class_method :handle_grpc_unavailable!
+
     def self.current_transaction_labels
       Gitlab::Metrics::Transaction.current&.labels || {}
     end

lib/gitlab/prometheus/metric_group.rb

@@ -6,9 +6,14 @@ module Gitlab
       attr_accessor :name, :priority, :metrics
       validates :name, :priority, :metrics, presence: true
 
-      def self.all
+      def self.common_metrics
         AdditionalMetricsParser.load_groups_from_yaml
       end
+
+      # EE only
+      def self.for_project(_)
+        common_metrics
+      end
     end
   end
 end

lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
+              deployment.project,
               common_query_context(
                 deployment.environment,
                 timeframe_start: (deployment.created_at - 30.minutes).to_f,

lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb

@@ -7,6 +7,7 @@ module Gitlab
         def query(environment_id)
           ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
+              environment.project,
               common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
             )
           end

lib/gitlab/prometheus/queries/matched_metrics_query.rb

@@ -18,7 +18,7 @@ module Gitlab
         private
 
         def groups_data
-          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all)
+          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics)
           lookup = active_series_lookup(metrics_groups)
 
           groups = {}

lib/gitlab/prometheus/queries/query_additional_metrics.rb

@@ -2,10 +2,10 @@ module Gitlab
   module Prometheus
     module Queries
       module QueryAdditionalMetrics
-        def query_metrics(query_context)
+        def query_metrics(project, query_context)
           query_processor = method(:process_query).curry[query_context]
 
-          groups = matched_metrics.map do |group|
+          groups = matched_metrics(project).map do |group|
             metrics = group.metrics.map do |metric|
               {
                 title: metric.title,
@@ -60,8 +60,8 @@ module Gitlab
           @available_metrics ||= client_label_values || []
         end
 
-        def matched_metrics
-          result = Gitlab::Prometheus::MetricGroup.all.map do |group|
+        def matched_metrics(project)
+          result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group|
             group.metrics.select! do |metric|
               metric.required_metrics.all?(&available_metrics.method(:include?))
             end

lib/gitlab/prometheus_client.rb

 module Gitlab
-  PrometheusError = Class.new(StandardError)
-
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
+    Error = Class.new(StandardError)
+    QueryError = Class.new(Gitlab::PrometheusClient::Error)
+
     attr_reader :rest_client, :headers
 
     def initialize(rest_client)
@@ -43,22 +44,22 @@ module Gitlab
       path = ['api', 'v1', type].join('/')
       get(path, args)
     rescue JSON::ParserError
-      raise PrometheusError, 'Parsing response failed'
+      raise PrometheusClient::Error, 'Parsing response failed'
     rescue Errno::ECONNREFUSED
-      raise PrometheusError, 'Connection refused'
+      raise PrometheusClient::Error, 'Connection refused'
     end
 
     def get(path, args)
       response = rest_client[path].get(params: args)
       handle_response(response)
     rescue SocketError
-      raise PrometheusError, "Can't connect to #{rest_client.url}"
+      raise PrometheusClient::Error, "Can't connect to #{rest_client.url}"
     rescue OpenSSL::SSL::SSLError
-      raise PrometheusError, "#{rest_client.url} contains invalid SSL data"
+      raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data"
     rescue RestClient::ExceptionWithResponse => ex
       handle_exception_response(ex.response)
     rescue RestClient::Exception
-      raise PrometheusError, "Network connection error"
+      raise PrometheusClient::Error, "Network connection error"
     end
 
     def handle_response(response)
@@ -66,16 +67,18 @@ module Gitlab
       if response.code == 200 && json_data['status'] == 'success'
         json_data['data'] || {}
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 
     def handle_exception_response(response)
-      if response.code == 400
+      if response.code == 200 && response['status'] == 'success'
+        response['data'] || {}
+      elsif response.code == 400
         json_data = JSON.parse(response.body)
-        raise PrometheusError, json_data['error'] || 'Bad data received'
+        raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received'
       else
-        raise PrometheusError, "#{response.code} - #{response.body}"
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
       end
     end
 

lib/gitlab/sidekiq_middleware/memory_killer.rb

-module Gitlab
-  module SidekiqMiddleware
-    class MemoryKiller
-      # Default the RSS limit to 0, meaning the MemoryKiller is disabled
-      MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
-      # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
-      GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
-      # Wait 30 seconds for running jobs to finish during graceful shutdown
-      SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
-
-      # Create a mutex used to ensure there will be only one thread waiting to
-      # shut Sidekiq down
-      MUTEX = Mutex.new
-
-      def call(worker, job, queue)
-        yield
-
-        current_rss = get_rss
-
-        return unless MAX_RSS > 0 && current_rss > MAX_RSS
-
-        Thread.new do
-          # Return if another thread is already waiting to shut Sidekiq down
-          return unless MUTEX.try_lock
-
-          Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\
-            " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}"
-          Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
-
-          # Wait `GRACE_TIME` to give the memory intensive job time to finish.
-          # Then, tell Sidekiq to stop fetching new jobs.
-          wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs')
-
-          # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
-          # Then, tell Sidekiq to gracefully shut down by giving jobs a few more
-          # moments to finish, killing and requeuing them if they didn't, and
-          # then terminating itself.
-          wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down')
-
-          # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
-          wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die')
-        end
-      end
-
-      private
-
-      def get_rss
-        output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
-        return 0 unless status.zero?
-
-        output.to_i
-      end
-
-      def wait_and_signal(time, signal, explanation)
-        Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
-        sleep(time)
-
-        Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
-        Process.kill(signal, pid)
-      end
-
-      def pid
-        Process.pid
-      end
-    end
-  end
-end