Merge branch 'ce-to-ee-2018-03-23' into 'master'

CE upstream - 2018-03-23 18:25 UTC

Closes gitaly#1101, #4683, and gitlab-com/infrastructure#19

See merge request gitlab-org/gitlab-ee!5102

.gitlab-ci.yml

@@ -625,7 +625,7 @@ migration:path-mysql:
 .db-rollback: &db-rollback
   <<: *dedicated-no-docs-pull-cache-job
   script:
-    - bundle exec rake db:rollback STEP=119
+    - bundle exec rake db:migrate VERSION=20170523121229
     - bundle exec rake db:migrate
 
 db:rollback-pg:

GITLAB_SHELL_VERSION

-7.1.0
+7.1.1

Gemfile

@@ -241,7 +241,7 @@ gem 'sanitize', '~> 2.0'
 gem 'babosa', '~> 1.0.2'
 
 # Sanitizes SVG input
-gem 'loofah', '~> 2.0.3'
+gem 'loofah', '~> 2.2'
 
 # Working with license
 gem 'licensee', '~> 8.9'

Gemfile.lock

@@ -151,6 +151,7 @@ GEM
     connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
+    crass (1.0.3)
     creole (0.5.0)
     css_parser (1.5.0)
       addressable
@@ -514,7 +515,8 @@ GEM
       actionpack (>= 4, < 5.2)
       activesupport (>= 4, < 5.2)
       railties (>= 4, < 5.2)
-    loofah (2.0.3)
+    loofah (2.2.2)
+      crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
       mini_mime (>= 0.1.1)
@@ -709,8 +711,8 @@ GEM
       activesupport (>= 4.2.0, < 5.0)
       nokogiri (~> 1.6)
       rails-deprecated_sanitizer (>= 1.0.1)
-    rails-html-sanitizer (1.0.3)
-      loofah (~> 2.0)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
     rails-i18n (4.0.9)
       i18n (~> 0.7)
       railties (~> 4.0)
@@ -1130,7 +1132,7 @@ DEPENDENCIES
   license_finder (~> 3.1)
   licensee (~> 8.9)
   lograge (~> 0.5)
-  loofah (~> 2.0.3)
+  loofah (~> 2.2)
   mail_room (~> 0.9.1)
   method_source (~> 0.8)
   minitest (~> 5.7.0)

app/assets/javascripts/performance_bar/services/performance_bar_service.js

+import Vue from 'vue';
+import _ from 'underscore';
 import axios from '../../lib/utils/axios_utils';
 
+let vueResourceInterceptor;
+
 export default class PerformanceBarService {
   static fetchRequestDetails(peekUrl, requestId) {
     return axios.get(peekUrl, { params: { request_id: requestId } });
   }
 
   static registerInterceptor(peekUrl, callback) {
+    vueResourceInterceptor = (request, next) => {
+      next(response => {
+        const requestId = response.headers['x-request-id'];
+        const requestUrl = response.url;
+
+        if (requestUrl !== peekUrl && requestId) {
+          callback(requestId, requestUrl);
+        }
+      });
+    };
+
+    Vue.http.interceptors.push(vueResourceInterceptor);
+
     return axios.interceptors.response.use(response => {
       const requestId = response.headers['x-request-id'];
       const requestUrl = response.config.url;
@@ -20,5 +37,9 @@ export default class PerformanceBarService {
 
   static removeInterceptor(interceptor) {
     axios.interceptors.response.eject(interceptor);
+    Vue.http.interceptors = _.without(
+      Vue.http.interceptors,
+      vueResourceInterceptor,
+    );
   }
 }

app/assets/stylesheets/performance_bar.scss

@@ -15,6 +15,10 @@
   line-height: $performance-bar-height;
   color: $perf-bar-text;
 
+  select {
+    width: 200px;
+  }
+
   &.disabled {
     display: none;
   }

app/controllers/omniauth_callbacks_controller.rb

@@ -19,6 +19,18 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController
     end
   end
 
+  # Extend the standard implementation to also increment
+  # the number of failed sign in attempts
+  def failure
+    if params[:username].present? && AuthHelper.form_based_provider?(failed_strategy.name)
+      user = User.by_login(params[:username])
+
+      user&.increment_failed_attempts!
+    end
+
+    super
+  end
+
   # Extend the standard message generation to accept our custom exception
   def failure_message
     exception = env["omniauth.error"]

app/controllers/projects/pages_controller.rb

@@ -21,4 +21,26 @@ class Projects::PagesController < Projects::ApplicationController
       end
     end
   end
+
+  def update
+    result = Projects::UpdateService.new(@project, current_user, project_params).execute
+
+    respond_to do |format|
+      format.html do
+        if result[:status] == :success
+          flash[:notice] = 'Your changes have been saved'
+        else
+          flash[:alert] = 'Something went wrong on our end'
+        end
+
+        redirect_to project_pages_path(@project)
+      end
+    end
+  end
+
+  private
+
+  def project_params
+    params.require(:project).permit(:pages_https_only)
+  end
 end

app/controllers/projects/pipelines_settings_controller.rb

@@ -10,10 +10,7 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController
       if service.execute
         flash[:notice] = "Pipelines settings for '#{@project.name}' were successfully updated."
 
-        if service.run_auto_devops_pipeline?
-          CreatePipelineWorker.perform_async(project.id, current_user.id, project.default_branch, :web, ignore_skip_ci: true, save_on_errors: false)
-          flash[:success] = "A new Auto DevOps pipeline has been created, go to <a href=\"#{project_pipelines_path(@project)}\">Pipelines page</a> for details".html_safe
-        end
+        run_autodevops_pipeline(service)
 
         redirect_to project_settings_ci_cd_path(@project)
       else
@@ -24,6 +21,18 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController
 
   private
 
+  def run_autodevops_pipeline(service)
+    return unless service.run_auto_devops_pipeline?
+
+    if @project.empty_repo?
+      flash[:warning] = "This repository is currently empty. A new Auto DevOps pipeline will be created after a new file has been pushed to a branch."
+      return
+    end
+
+    CreatePipelineWorker.perform_async(project.id, current_user.id, project.default_branch, :web, ignore_skip_ci: true, save_on_errors: false)
+    flash[:success] = "A new Auto DevOps pipeline has been created, go to <a href=\"#{project_pipelines_path(@project)}\">Pipelines page</a> for details".html_safe
+  end
+
   def update_params
     params.require(:project).permit(
       :runners_token, :builds_enabled, :build_allow_git_fetch,

app/helpers/application_settings_helper.rb

@@ -97,7 +97,7 @@ module ApplicationSettingsHelper
 
   def repository_storages_options_for_select(selected)
     options = Gitlab.config.repositories.storages.map do |name, storage|
-      ["#{name} - #{storage.legacy_disk_path}", name]
+      ["#{name} - #{storage['gitaly_address']}", name]
     end
 
     options_for_select(options, selected)

app/helpers/projects_helper.rb

@@ -582,4 +582,22 @@ module ProjectsHelper
   def can_show_last_commit_in_list?(project)
     can?(current_user, :read_cross_project) && project.commit
   end
+
+  def pages_https_only_disabled?
+    !@project.pages_domains.all?(&:https?)
+  end
+
+  def pages_https_only_title
+    return unless pages_https_only_disabled?
+
+    "You must enable HTTPS for all your domains first"
+  end
+
+  def pages_https_only_label_class
+    if pages_https_only_disabled?
+      "list-label disabled"
+    else
+      "list-label"
+    end
+  end
 end

app/models/pages_domain.rb

@@ -6,8 +6,10 @@ class PagesDomain < ActiveRecord::Base
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }
-  validates :certificate, certificate: true, allow_nil: true, allow_blank: true
-  validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+  validates :certificate, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? }
+  validates :certificate, certificate: true, if: ->(domain) { domain.certificate.present? }
+  validates :key, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? }
+  validates :key, certificate_key: true, if: ->(domain) { domain.key.present? }
   validates :verification_code, presence: true, allow_blank: false
 
   validate :validate_pages_domain
@@ -46,6 +48,10 @@ class PagesDomain < ActiveRecord::Base
     !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present?
   end
 
+  def https?
+    certificate.present?
+  end
+
   def to_param
     domain
   end

app/models/project.rb

@@ -274,6 +274,7 @@ class Project < ActiveRecord::Base
   validate :visibility_level_allowed_by_group
   validate :visibility_level_allowed_as_fork
   validate :check_wiki_path_conflict
+  validate :validate_pages_https_only, if: -> { changes.has_key?(:pages_https_only) }
   validates :repository_storage,
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
@@ -750,6 +751,26 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def pages_https_only
+    return false unless Gitlab.config.pages.external_https
+
+    super
+  end
+
+  def pages_https_only?
+    return false unless Gitlab.config.pages.external_https
+
+    super
+  end
+
+  def validate_pages_https_only
+    return unless pages_https_only?
+
+    unless pages_domains.all?(&:https?)
+      errors.add(:pages_https_only, "cannot be enabled unless all domains have TLS certificates")
+    end
+  end
+
   def to_param
     if persisted? && errors.include?(:path)
       path_was

app/services/projects/update_pages_configuration_service.rb

@@ -18,7 +18,8 @@ module Projects
 
     def pages_config
       {
-        domains: pages_domains_config
+        domains: pages_domains_config,
+        https_only: project.pages_https_only?
       }
     end
 
@@ -27,7 +28,8 @@ module Projects
         {
           domain: domain.domain,
           certificate: domain.certificate,
-          key: domain.key
+          key: domain.key,
+          https_only: project.pages_https_only? && domain.https?
         }
       end
     end

app/services/projects/update_service.rb

@@ -34,6 +34,8 @@ module Projects
           system_hook_service.execute_hooks_for(project, :update)
         end
 
+        update_pages_config if changing_pages_https_only?
+
         success
       else
         model_errors = project.errors.full_messages.to_sentence
@@ -77,5 +79,13 @@ module Projects
       log_error("Could not create wiki for #{project.full_name}")
       Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki')
     end
+
+    def update_pages_config
+      Projects::UpdatePagesConfigurationService.new(project).execute
+    end
+
+    def changing_pages_https_only?
+      project.previous_changes.include?(:pages_https_only)
+    end
   end
 end

app/validators/certificate_validator.rb

@@ -16,8 +16,6 @@ class CertificateValidator < ActiveModel::EachValidator
   private
 
   def valid_certificate_pem?(value)
-    return false unless value
-
     OpenSSL::X509::Certificate.new(value).present?
   rescue OpenSSL::X509::CertificateError
     false

app/views/projects/pages/_https_only.html.haml

+= form_for @project, url: namespace_project_pages_path(@project.namespace.becomes(Namespace), @project), html: { class: 'inline', title: pages_https_only_title } do |f|
+  = f.check_box :pages_https_only, class: 'pull-left', disabled: pages_https_only_disabled?
+
+  .prepend-left-20
+    = f.label :pages_https_only, class: pages_https_only_label_class do
+      %strong Force domains with SSL certificates to use HTTPS
+
+  - unless pages_https_only_disabled?
+    .prepend-top-10
+    = f.submit 'Save', class: 'btn btn-success'

app/views/projects/pages/show.html.haml

@@ -13,6 +13,9 @@
   Combined with the power of GitLab CI and the help of GitLab Runner
   you can deploy static pages for your individual projects, your user or your group.
 
+- if Gitlab.config.pages.external_https
+  = render 'https_only'
+
 %hr.clearfix
 
 = render 'access'

changelogs/unreleased/43482-enabling-auto-devops-on-an-empty-project-gives-you-wrong-information.yml

+---
+title: Add empty repo check before running AutoDevOps pipeline
+merge_request: 17605
+author:
+type: changed

changelogs/unreleased/43525-limit-number-of-failed-logins-using-ldap.yml

+---
+title: Limit the number of failed logins when using LDAP for authentication
+merge_request: 43525
+author:
+type: added

changelogs/unreleased/44587-autolinking-includes-trailing-exclamation-marks.yml

+---
+title: Don't capture trailing punctuation when autolinking
+merge_request: 17965
+author:
+type: fixed

changelogs/unreleased/increase-unicorn-memory-killer-limits.yml

+---
+title: Increase the memory limits used in the unicorn killer
+merge_request: 17948
+author:
+type: other

changelogs/unreleased/jprovazn-issueref.yml

+---
+title: Display state indicator for issuable references in non-project scope (e.g.
+  when referencing issuables from group scope).
+merge_request:
+author:
+type: fixed

changelogs/unreleased/pages_force_https.yml

+---
+title: Add HTTPS-only pages
+merge_request: 16273
+author: rfwatson
+type: added

changelogs/unreleased/sh-update-loofah.yml

+---
+title: Bump rails-html-sanitizer to 1.0.4
+merge_request:
+author:
+type: security

config.ru

@@ -7,8 +7,8 @@ if defined?(Unicorn)
     # Unicorn self-process killer
     require 'unicorn/worker_killer'
 
-    min = (ENV['GITLAB_UNICORN_MEMORY_MIN'] || 300 * 1 << 20).to_i
-    max = (ENV['GITLAB_UNICORN_MEMORY_MAX'] || 350 * 1 << 20).to_i
+    min = (ENV['GITLAB_UNICORN_MEMORY_MIN'] || 400 * 1 << 20).to_i
+    max = (ENV['GITLAB_UNICORN_MEMORY_MAX'] || 650 * 1 << 20).to_i
 
     # Max memory size (RSS) per worker
     use Unicorn::WorkerKiller::Oom, min, max

config/routes/project.rb

@@ -52,7 +52,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
-      resource :pages, only: [:show, :destroy] do
+      resource :pages, only: [:show, :update, :destroy] do
         resources :domains, except: :index, controller: 'pages_domains', constraints: { id: %r{[^/]+} } do
           member do
             post :verify

db/migrate/20180102220145_add_pages_https_only_to_projects.rb

+class AddPagesHttpsOnlyToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :pages_https_only, :boolean
+  end
+end

db/migrate/20180109183319_change_default_value_for_pages_https_only.rb

+class ChangeDefaultValueForPagesHttpsOnly < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    change_column_default :projects, :pages_https_only, true
+  end
+
+  def down
+    change_column_default :projects, :pages_https_only, nil
+  end
+end

db/schema.rb

@@ -1994,6 +1994,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do
     t.boolean "mirror_overwrites_diverged_branches"
     t.string "external_authorization_classification_label"
     t.string "external_webhook_token"
+    t.boolean "pages_https_only", default: true
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree

doc/development/i18n/proofreader.md

@@ -10,6 +10,7 @@ are very appreciative of the work done by translators and proofreaders!
   - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
 - Chinese Traditional
   - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
+  - Weizhe Ding - [GitLab](https://gitlab.com/d.weizhe), [Crowdin](https://crowdin.com/profile/d.weizhe)
 - Chinese Traditional, Hong Kong
   - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve)
 - Dutch

doc/user/project/pipelines/schedules.md

@@ -12,7 +12,7 @@ month on the 22nd for a certain branch.
 
 In order to schedule a pipeline:
 
-1. Navigate to your project's **Pipelines ➔ Schedules** and click the
+1. Navigate to your project's **CI / CD ➔ Schedules** and click the
    **New Schedule** button.
 1. Fill in the form
 1. Hit **Save pipeline schedule** for the changes to take effect.

lib/banzai/filter/autolink_filter.rb

@@ -21,12 +21,13 @@ module Banzai
       #
       # See http://en.wikipedia.org/wiki/URI_scheme
       #
-      # The negative lookbehind ensures that users can paste a URL followed by a
-      # period or comma for punctuation without those characters being included
-      # in the generated link.
+      # The negative lookbehind ensures that users can paste a URL followed by
+      # punctuation without those characters being included in the generated
+      # link. It matches the behaviour of Rinku 2.0.1:
+      # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65
       #
-      # Rubular: http://rubular.com/r/JzPhi6DCZp
-      LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)}
+      # Rubular: http://rubular.com/r/nrL3r9yUiq
+      LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)}
 
       # Text matching LINK_PATTERN inside these elements will not be linked
       IGNORE_PARENTS = %w(a code kbd pre script style).to_set

lib/banzai/filter/issuable_state_filter.rb

@@ -17,7 +17,7 @@ module Banzai
         issuables.each do |node, issuable|
           next if !can_read_cross_project? && issuable.project != project
 
-          if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project)
+          if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable)
             node.content += " (#{issuable.state})"
           end
         end
@@ -27,6 +27,10 @@ module Banzai
 
       private
 
+      def issuable_reference?(text, issuable)
+        text == issuable.reference_link_text(project || group)
+      end
+
       def can_read_cross_project?
         Ability.allowed?(current_user, :read_cross_project)
       end
@@ -38,6 +42,10 @@ module Banzai
       def project
         context[:project]
       end
+
+      def group
+        context[:group]
+      end
     end
   end
 end

lib/gitlab/encoding_helper.rb

@@ -90,7 +90,7 @@ module Gitlab
     end
 
     def clean(message)
-      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "".encode("UTF-16BE"))
         .encode("UTF-8")
         .gsub("\0".encode("UTF-8"), "")
     end

spec/controllers/omniauth_callbacks_controller_spec.rb

@@ -10,6 +10,41 @@ describe OmniauthCallbacksController do
     stub_omniauth_provider(provider, context: request)
   end
 
+  context 'when the user is on the last sign in attempt' do
+    let(:extern_uid) { 'my-uid' }
+
+    before do
+      user.update(failed_attempts: User.maximum_attempts.pred)
+      subject.response = ActionDispatch::Response.new
+    end
+
+    context 'when using a form based provider' do
+      let(:provider) { :ldap }
+
+      it 'locks the user when sign in fails' do
+        allow(subject).to receive(:params).and_return(ActionController::Parameters.new(username: user.username))
+        request.env['omniauth.error.strategy'] = OmniAuth::Strategies::LDAP.new(nil)
+
+        subject.send(:failure)
+
+        expect(user.reload).to be_access_locked
+      end
+    end
+
+    context 'when using a button based provider' do
+      let(:provider) { :github }
+
+      it 'does not lock the user when sign in fails' do
+        request.env['omniauth.error.strategy'] = OmniAuth::Strategies::GitHub.new(nil)
+
+        subject.send(:failure)
+
+        expect(user.reload).not_to be_access_locked
+      end
+    end
+  end
+
+  context 'strategies' do
     context 'github' do
       let(:extern_uid) { 'my-uid' }
       let(:provider) { :github }
@@ -89,4 +124,5 @@ describe OmniauthCallbacksController do
         expect(controller).to set_flash[:alert].to('Wrong extern UID provided. Make sure Auth0 is configured correctly.')
       end
     end
+  end
 end

spec/controllers/projects/pages_controller_spec.rb

@@ -65,4 +65,41 @@ describe Projects::PagesController do
       end
     end
   end
+
+  describe 'PATCH update' do
+    let(:request_params) do
+      {
+        namespace_id: project.namespace,
+        project_id: project,
+        project: { pages_https_only: false }
+      }
+    end
+
+    let(:update_service) { double(execute: { status: :success }) }
+
+    before do
+      allow(Projects::UpdateService).to receive(:new) { update_service }
+    end
+
+    it 'returns 302 status' do
+      patch :update, request_params
+
+      expect(response).to have_gitlab_http_status(:found)
+    end
+
+    it 'redirects back to the pages settings' do
+      patch :update, request_params
+
+      expect(response).to redirect_to(project_pages_path(project))
+    end
+
+    it 'calls the update service' do
+      expect(Projects::UpdateService)
+        .to receive(:new)
+        .with(project, user, request_params[:project])
+        .and_return(update_service)
+
+      patch :update, request_params
+    end
+  end
 end

spec/controllers/projects/pages_domains_controller_spec.rb

@@ -13,7 +13,7 @@ describe Projects::PagesDomainsController do
   end
 
   let(:pages_domain_params) do
-    build(:pages_domain, :with_certificate, :with_key, domain: 'my.otherdomain.com').slice(:key, :certificate, :domain)
+    build(:pages_domain, domain: 'my.otherdomain.com').slice(:key, :certificate, :domain)
   end
 
   before do
@@ -68,7 +68,7 @@ describe Projects::PagesDomainsController do
     end
 
     let(:pages_domain_params) do
-      attributes_for(:pages_domain, :with_certificate, :with_key).slice(:key, :certificate)
+      attributes_for(:pages_domain).slice(:key, :certificate)
     end
 
     let(:params) do

spec/controllers/projects/pipelines_settings_controller_spec.rb

@@ -47,12 +47,34 @@ describe Projects::PipelinesSettingsController do
           expect_any_instance_of(Projects::UpdateService).to receive(:run_auto_devops_pipeline?).and_return(true)
         end
 
+        context 'when the project repository is empty' do
+          it 'sets a warning flash' do
+            expect(subject).to set_flash[:warning]
+          end
+
+          it 'does not queue a CreatePipelineWorker' do
+            expect(CreatePipelineWorker).not_to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args)
+
+            subject
+          end
+        end
+
+        context 'when the project repository is not empty' do
+          let(:project) { create(:project, :repository) }
+
+          it 'sets a success flash' do
+            allow(CreatePipelineWorker).to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args)
+
+            expect(subject).to set_flash[:success]
+          end
+
           it 'queues a CreatePipelineWorker' do
             expect(CreatePipelineWorker).to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args)
 
             subject
           end
         end
+      end
 
       context 'when run_auto_devops_pipeline is not true' do
         before do

spec/factories/pages_domains.rb

@@ -4,24 +4,6 @@ FactoryBot.define do
     verified_at { Time.now }
     enabled_until { 1.week.from_now }
 
-    trait :disabled do
-      verified_at nil
-      enabled_until nil
-    end
-
-    trait :unverified do
-      verified_at nil
-    end
-
-    trait :reverify do
-      enabled_until { 1.hour.from_now }
-    end
-
-    trait :expired do
-      enabled_until { 1.hour.ago }
-    end
-
-    trait :with_certificate do
     certificate '-----BEGIN CERTIFICATE-----
 MIICGzCCAYSgAwIBAgIBATANBgkqhkiG9w0BAQUFADAbMRkwFwYDVQQDExB0ZXN0
 LWNlcnRpZmljYXRlMB4XDTE2MDIxMjE0MzIwMFoXDTIwMDQxMjE0MzIwMFowGzEZ
@@ -36,9 +18,7 @@ joZp2JHYvNlTPkRJ/J4TcXxBTJmArcQgTIuNoBtC+0A/SwdK4MfTCUY4vNWNdese
 5A4K65Nb7Oh1AdQieTBHNXXCdyFsva9/ScfQGEl7p55a52jOPs0StPd7g64uvjlg
 YHi2yesCrOvVXt+lgPTd
 -----END CERTIFICATE-----'
-    end
 
-    trait :with_key do
     key '-----BEGIN PRIVATE KEY-----
 MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKS+CfS9GcRSdYSN
 SzyH5QJQBr5umRL6E+KilOV39iYFO/9oHjUdapTRWkrwnNPCp7qaeck4Jr8iv14t
@@ -55,6 +35,30 @@ EPjGlXIT+aW2XiPmK3ZlCDcWIenE+lmtbOpI159Wpk8BGXs/s/xBAkEAlAY3ymgx
 63BDJEwvOb2IaP8lDDxNsXx9XJNVvQbv5n15vNsLHbjslHfAhAbxnLQ1fLhUPqSi
 nNp/xedE1YxutQ==
 -----END PRIVATE KEY-----'
+
+    trait :disabled do
+      verified_at nil
+      enabled_until nil
+    end
+
+    trait :unverified do
+      verified_at nil
+    end
+
+    trait :reverify do
+      enabled_until { 1.hour.from_now }
+    end
+
+    trait :expired do
+      enabled_until { 1.hour.ago }
+    end
+
+    trait :without_certificate do
+      certificate nil
+    end
+
+    trait :without_key do
+      key nil
     end
 
     trait :with_missing_chain do

spec/features/projects/pages_spec.rb

@@ -40,11 +40,6 @@ feature 'Pages' do
       end
 
       context 'when support for external domains is disabled' do
-        before do
-          allow(Gitlab.config.pages).to receive(:external_http).and_return(nil)
-          allow(Gitlab.config.pages).to receive(:external_https).and_return(nil)
-        end
-
         it 'renders message that support is disabled' do
           visit project_pages_path(project)
 
@@ -52,7 +47,9 @@ feature 'Pages' do
         end
       end
 
-      context 'when pages are exposed on external HTTP address' do
+      context 'when pages are exposed on external HTTP address', :http_pages_enabled do
+        given(:project) { create(:project, pages_https_only: false) }
+
         shared_examples 'adds new domain' do
           it 'adds new domain' do
             visit new_project_pages_domain_path(project)
@@ -64,11 +61,6 @@ feature 'Pages' do
           end
         end
 
-        before do
-          allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80'])
-          allow(Gitlab.config.pages).to receive(:external_https).and_return(nil)
-        end
-
         it 'allows to add new domain' do
           visit project_pages_path(project)
 
@@ -80,13 +72,13 @@ feature 'Pages' do
         context 'when project in group namespace' do
           it_behaves_like 'adds new domain' do
             let(:group) { create :group }
-            let(:project) { create :project, namespace: group }
+            let(:project) { create(:project, namespace: group, pages_https_only: false) }
           end
         end
 
         context 'when pages domain is added' do
           before do
-            project.pages_domains.create!(domain: 'my.test.domain.com')
+            create(:pages_domain, project: project, domain: 'my.test.domain.com')
 
             visit new_project_pages_domain_path(project)
           end
@@ -104,7 +96,7 @@ feature 'Pages' do
         end
       end
 
-      context 'when pages are exposed on external HTTPS address' do
+      context 'when pages are exposed on external HTTPS address', :https_pages_enabled do
         let(:certificate_pem) do
           <<~PEM
           -----BEGIN CERTIFICATE-----
@@ -145,11 +137,6 @@ feature 'Pages' do
           KEY
         end
 
-        before do
-          allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80'])
-          allow(Gitlab.config.pages).to receive(:external_https).and_return(['1.1.1.1:443'])
-        end
-
         it 'adds new domain with certificate' do
           visit new_project_pages_domain_path(project)
 
@@ -163,7 +150,7 @@ feature 'Pages' do
 
         describe 'updating the certificate for an existing domain' do
           let!(:domain) do
-            create(:pages_domain, :with_key, :with_certificate, project: project)
+            create(:pages_domain, project: project)
           end
 
           it 'allows the certificate to be updated' do
@@ -237,6 +224,70 @@ feature 'Pages' do
     it_behaves_like 'no pages deployed'
   end
 
+  describe 'HTTPS settings', :js, :https_pages_enabled do
+    background do
+      project.namespace.update(owner: user)
+
+      allow_any_instance_of(Project).to receive(:pages_deployed?) { true }
+    end
+
+    scenario 'tries to change the setting' do
+      visit project_pages_path(project)
+      expect(page).to have_content("Force domains with SSL certificates to use HTTPS")
+
+      uncheck :project_pages_https_only
+
+      click_button 'Save'
+
+      expect(page).to have_text('Your changes have been saved')
+      expect(page).not_to have_checked_field('project_pages_https_only')
+    end
+
+    context 'setting could not be updated' do
+      let(:service) { instance_double('Projects::UpdateService') }
+
+      before do
+        allow(Projects::UpdateService).to receive(:new).and_return(service)
+        allow(service).to receive(:execute).and_return(status: :error)
+      end
+
+      scenario 'tries to change the setting' do
+        visit project_pages_path(project)
+
+        uncheck :project_pages_https_only
+
+        click_button 'Save'
+
+        expect(page).to have_text('Something went wrong on our end')
+      end
+    end
+
+    context 'non-HTTPS domain exists' do
+      given(:project) { create(:project, pages_https_only: false) }
+
+      before do
+        create(:pages_domain, :without_key, :without_certificate, project: project)
+      end
+
+      scenario 'the setting is disabled' do
+        visit project_pages_path(project)
+
+        expect(page).to have_field(:project_pages_https_only, disabled: true)
+        expect(page).not_to have_button('Save')
+      end
+    end
+
+    context 'HTTPS pages are disabled', :https_pages_disabled do
+      scenario 'the setting is unavailable' do
+        visit project_pages_path(project)
+
+        expect(page).not_to have_field(:project_pages_https_only)
+        expect(page).not_to have_content('Force domains with SSL certificates to use HTTPS')
+        expect(page).not_to have_button('Save')
+      end
+    end
+  end
+
   describe 'Remove page' do
     context 'when user is the owner' do
       let(:project) { create :project, :repository }

spec/initializers/6_validations_spec.rb

@@ -12,18 +12,14 @@ describe '6_validations' do
     FileUtils.rm_rf('tmp/tests/paths')
   end
 
+  describe 'validate_storages_config' do
     context 'with correct settings' do
       before do
         mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c'), 'bar' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/d'))
       end
 
-    context 'when one of the settings is incorrect' do
-      before do
-        mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c', 'failure_count_threshold' => 'not a number'))
-      end
-
-      it 'throws an error' do
-        expect { validate_storages_config }.to raise_error(/failure_count_threshold/)
+      it 'passes through' do
+        expect { validate_storages_config }.not_to raise_error
       end
     end
 
@@ -111,26 +107,6 @@ describe '6_validations' do
     end
   end
 
-  context 'with incomplete settings' do
-    before do
-      mock_storages('foo' => {})
-    end
-
-    it 'throws an error suggesting the user to update its settings' do
-      expect { validate_storages_config }.to raise_error('foo is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example. Please fix this in your gitlab.yml before starting GitLab.')
-    end
-  end
-
-  context 'with deprecated settings structure' do
-    before do
-      mock_storages('foo' => 'tmp/tests/paths/a/b/c')
-    end
-
-    it 'throws an error suggesting the user to update its settings' do
-      expect { validate_storages_config }.to raise_error("foo is not a valid storage, because it has no `path` key. It may be configured as:\n\nfoo:\n  path: tmp/tests/paths/a/b/c\n\nFor source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example.\n\nIf you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`.\n")
-    end
-  end
-
   def mock_storages(storages)
     allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
   end

spec/lib/banzai/filter/autolink_filter_spec.rb

@@ -122,15 +122,11 @@ describe Banzai::Filter::AutolinkFilter do
     end
 
     it 'does not include trailing punctuation' do
-      doc = filter("See #{link}.")
-      expect(doc.at_css('a').text).to eq link
-
-      doc = filter("See #{link}, ok?")
-      expect(doc.at_css('a').text).to eq link
-
-      doc = filter("See #{link}...")
+      ['.', ', ok?', '...', '?', '!', ': is that ok?'].each do |trailing_punctuation|
+        doc = filter("See #{link}#{trailing_punctuation}")
         expect(doc.at_css('a').text).to eq link
       end
+    end
 
     it 'includes trailing punctuation when part of a balanced pair' do
       described_class::PUNCTUATION_PAIRS.each do |close, open|

spec/lib/banzai/filter/issuable_state_filter_spec.rb

@@ -8,6 +8,7 @@ describe Banzai::Filter::IssuableStateFilter do
   let(:context) { { current_user: user, issuable_state_filter_enabled: true } }
   let(:closed_issue) { create_issue(:closed) }
   let(:project) { create(:project, :public) }
+  let(:group) { create(:group) }
   let(:other_project) { create(:project, :public) }
 
   def create_link(text, data)
@@ -77,6 +78,13 @@ describe Banzai::Filter::IssuableStateFilter do
     expect(doc.css('a').last.text).to eq("#{closed_issue.to_reference(other_project)} (closed)")
   end
 
+  it 'handles references from group scopes' do
+    link = create_link(closed_issue.to_reference(other_project), issue: closed_issue.id, reference_type: 'issue')
+    doc = filter(link, context.merge(project: nil, group: group))
+
+    expect(doc.css('a').last.text).to eq("#{closed_issue.to_reference(other_project)} (closed)")
+  end
+
   it 'skips cross project references if the user cannot read cross project' do
     expect(Ability).to receive(:allowed?).with(user, :read_cross_project) { false }
     link = create_link(closed_issue.to_reference(other_project), issue: closed_issue.id, reference_type: 'issue')

spec/lib/gitlab/encoding_helper_spec.rb

@@ -161,6 +161,11 @@ describe Gitlab::EncodingHelper do
         'removes invalid bytes from ASCII-8bit encoded multibyte string.',
         "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'),
         "Lorem ipsum\n dolor sit amet, xyàyùabcdùefg"
+      ],
+      [
+        'handles UTF-16BE encoded strings',
+        "\xFE\xFF\x00\x41".force_encoding('ASCII-8BIT'), # An "A" prepended with UTF-16 BOM
+        "\xEF\xBB\xBFA" # An "A" prepended with UTF-8 BOM
       ]
     ].each do |description, test_string, xpect|
       it description do

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -476,6 +476,7 @@ Project:
 - jobs_cache_index
 - external_authorization_classification_label
 - external_webhook_token
+- pages_https_only
 Author:
 - name
 ProjectFeature:

spec/models/pages_domain_spec.rb

@@ -18,6 +18,7 @@ describe PagesDomain do
       it { is_expected.to validate_uniqueness_of(:domain).case_insensitive }
     end
 
+    describe "hostname" do
       {
         'my.domain.com'    => true,
         '123.456.789'      => true,
@@ -40,29 +41,67 @@ describe PagesDomain do
       end
     end
 
+    describe "HTTPS-only" do
+      using RSpec::Parameterized::TableSyntax
+
+      let(:domain) { 'my.domain.com' }
+
+      let(:project) do
+        instance_double(Project, pages_https_only?: pages_https_only)
+      end
+
+      let(:pages_domain) do
+        build(:pages_domain, certificate: certificate, key: key).tap do |pd|
+          allow(pd).to receive(:project).and_return(project)
+          pd.valid?
+        end
+      end
+
+      where(:pages_https_only, :certificate, :key, :errors_on) do
+        attributes = attributes_for(:pages_domain)
+        cert, key = attributes.fetch_values(:certificate, :key)
+
+        true  | nil  | nil | %i(certificate key)
+        true  | cert | nil | %i(key)
+        true  | nil  | key | %i(certificate key)
+        true  | cert | key | []
+        false | nil  | nil | []
+        false | cert | nil | %i(key)
+        false | nil  | key | %i(key)
+        false | cert | key | []
+      end
+
+      with_them do
+        it "is adds the expected errors" do
+          expect(pages_domain.errors.keys).to eq errors_on
+        end
+      end
+    end
+  end
+
   describe 'validate certificate' do
     subject { domain }
 
-    context 'when only certificate is specified' do
-      let(:domain) { build(:pages_domain, :with_certificate) }
+    context 'with matching key' do
+      let(:domain) { build(:pages_domain) }
 
-      it { is_expected.not_to be_valid }
+      it { is_expected.to be_valid }
     end
 
-    context 'when only key is specified' do
-      let(:domain) { build(:pages_domain, :with_key) }
+    context 'when no certificate is specified' do
+      let(:domain) { build(:pages_domain, :without_certificate) }
 
       it { is_expected.not_to be_valid }
     end
 
-    context 'with matching key' do
-      let(:domain) { build(:pages_domain, :with_certificate, :with_key) }
+    context 'when no key is specified' do
+      let(:domain) { build(:pages_domain, :without_key) }
 
-      it { is_expected.to be_valid }
+      it { is_expected.not_to be_valid }
     end
 
     context 'for not matching key' do
-      let(:domain) { build(:pages_domain, :with_missing_chain, :with_key) }
+      let(:domain) { build(:pages_domain, :with_missing_chain) }
 
       it { is_expected.not_to be_valid }
     end
@@ -103,30 +142,26 @@ describe PagesDomain do
   describe '#url' do
     subject { domain.url }
 
-    context 'without the certificate' do
-      let(:domain) { build(:pages_domain, certificate: '') }
+    let(:domain) { build(:pages_domain) }
 
-      it { is_expected.to eq("http://#{domain.domain}") }
-    end
+    it { is_expected.to eq("https://#{domain.domain}") }
 
-    context 'with a certificate' do
-      let(:domain) { build(:pages_domain, :with_certificate) }
+    context 'without the certificate' do
+      let(:domain) { build(:pages_domain, :without_certificate) }
 
-      it { is_expected.to eq("https://#{domain.domain}") }
+      it { is_expected.to eq("http://#{domain.domain}") }
     end
   end
 
   describe '#has_matching_key?' do
     subject { domain.has_matching_key? }
 
-    context 'for matching key' do
-      let(:domain) { build(:pages_domain, :with_certificate, :with_key) }
+    let(:domain) { build(:pages_domain) }
 
     it { is_expected.to be_truthy }
-    end
 
     context 'for invalid key' do
-      let(:domain) { build(:pages_domain, :with_missing_chain, :with_key) }
+      let(:domain) { build(:pages_domain, :with_missing_chain) }
 
       it { is_expected.to be_falsey }
     end
@@ -136,7 +171,7 @@ describe PagesDomain do
     subject { domain.has_intermediates? }
 
     context 'for self signed' do
-      let(:domain) { build(:pages_domain, :with_certificate) }
+      let(:domain) { build(:pages_domain) }
 
       it { is_expected.to be_truthy }
     end
@@ -162,7 +197,7 @@ describe PagesDomain do
     subject { domain.expired? }
 
     context 'for valid' do
-      let(:domain) { build(:pages_domain, :with_certificate) }
+      let(:domain) { build(:pages_domain) }
 
       it { is_expected.to be_falsey }
     end
@@ -175,7 +210,7 @@ describe PagesDomain do
   end
 
   describe '#subject' do
-    let(:domain) { build(:pages_domain, :with_certificate) }
+    let(:domain) { build(:pages_domain) }
 
     subject { domain.subject }
 
@@ -183,7 +218,7 @@ describe PagesDomain do
   end
 
   describe '#certificate_text' do
-    let(:domain) { build(:pages_domain, :with_certificate) }
+    let(:domain) { build(:pages_domain) }
 
     subject { domain.certificate_text }
 
@@ -191,6 +226,18 @@ describe PagesDomain do
     it { is_expected.not_to be_empty }
   end
 
+  describe "#https?" do
+    context "when a certificate is present" do
+      subject { build(:pages_domain) }
+      it { is_expected.to be_https }
+    end
+
+    context "when no certificate is present" do
+      subject { build(:pages_domain, :without_certificate) }
+      it { is_expected.not_to be_https }
+    end
+  end
+
   describe '#update_daemon' do
     it 'runs when the domain is created' do
       domain = build(:pages_domain)
@@ -267,29 +314,30 @@ describe PagesDomain do
       end
 
       context 'TLS configuration' do
-        set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) }
+        set(:domain_without_tls) { create(:pages_domain, :without_certificate, :without_key) }
+        set(:domain) { create(:pages_domain) }
 
-        let(:cert1) { domain_with_tls.certificate }
+        let(:cert1) { domain.certificate }
         let(:cert2) { cert1 + ' ' }
-        let(:key1) { domain_with_tls.key }
+        let(:key1) { domain.key }
         let(:key2) { key1 + ' ' }
 
         it 'updates when added' do
-          expect(domain).to receive(:update_daemon)
+          expect(domain_without_tls).to receive(:update_daemon)
 
-          domain.update!(key: key1, certificate: cert1)
+          domain_without_tls.update!(key: key1, certificate: cert1)
         end
 
         it 'updates when changed' do
-          expect(domain_with_tls).to receive(:update_daemon)
+          expect(domain).to receive(:update_daemon)
 
-          domain_with_tls.update!(key: key2, certificate: cert2)
+          domain.update!(key: key2, certificate: cert2)
         end
 
         it 'updates when removed' do
-          expect(domain_with_tls).to receive(:update_daemon)
+          expect(domain).to receive(:update_daemon)
 
-          domain_with_tls.update!(key: nil, certificate: nil)
+          domain.update!(key: nil, certificate: nil)
         end
       end
     end

spec/models/project_spec.rb

@@ -3873,4 +3873,49 @@ describe Project do
       end
     end
   end
+
+  describe "#pages_https_only?" do
+    subject { build(:project) }
+
+    context "when HTTPS pages are disabled" do
+      it { is_expected.not_to be_pages_https_only }
+    end
+
+    context "when HTTPS pages are enabled", :https_pages_enabled do
+      it { is_expected.to be_pages_https_only }
+    end
+  end
+
+  describe "#pages_https_only? validation", :https_pages_enabled do
+    subject(:project) do
+      # set-up dirty object:
+      create(:project, pages_https_only: false).tap do |p|
+        p.pages_https_only = true
+      end
+    end
+
+    context "when no domains are associated" do
+      it { is_expected.to be_valid }
+    end
+
+    context "when domains including keys and certificates are associated" do
+      before do
+        allow(project)
+          .to receive(:pages_domains)
+          .and_return([instance_double(PagesDomain, https?: true)])
+      end
+
+      it { is_expected.to be_valid }
+    end
+
+    context "when domains including no keys or certificates are associated" do
+      before do
+        allow(project)
+          .to receive(:pages_domains)
+          .and_return([instance_double(PagesDomain, https?: false)])
+      end
+
+      it { is_expected.not_to be_valid }
+    end
+  end
 end

spec/requests/api/pages_domains_spec.rb

 require 'rails_helper'
 
 describe API::PagesDomains do
-  set(:project) { create(:project, path: 'my.project') }
+  set(:project) { create(:project, path: 'my.project', pages_https_only: false) }
   set(:user) { create(:user) }
   set(:admin) { create(:admin) }
 
-  set(:pages_domain) { create(:pages_domain, domain: 'www.domain.test', project: project) }
-  set(:pages_domain_secure) { create(:pages_domain, :with_certificate, :with_key, domain: 'ssl.domain.test', project: project) }
-  set(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, :with_key, domain: 'expired.domain.test', project: project) }
+  set(:pages_domain) { create(:pages_domain, :without_key, :without_certificate, domain: 'www.domain.test', project: project) }
+  set(:pages_domain_secure) { create(:pages_domain, domain: 'ssl.domain.test', project: project) }
+  set(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, domain: 'expired.domain.test', project: project) }
 
-  let(:pages_domain_params) { build(:pages_domain, domain: 'www.other-domain.test').slice(:domain) }
-  let(:pages_domain_secure_params) { build(:pages_domain, :with_certificate, :with_key, domain: 'ssl.other-domain.test', project: project).slice(:domain, :certificate, :key) }
-  let(:pages_domain_secure_key_missmatch_params) {build(:pages_domain, :with_trusted_chain, :with_key, project: project).slice(:domain, :certificate, :key) }
+  let(:pages_domain_params) { build(:pages_domain, :without_key, :without_certificate, domain: 'www.other-domain.test').slice(:domain) }
+  let(:pages_domain_secure_params) { build(:pages_domain, domain: 'ssl.other-domain.test', project: project).slice(:domain, :certificate, :key) }
+  let(:pages_domain_secure_key_missmatch_params) {build(:pages_domain, :with_trusted_chain, project: project).slice(:domain, :certificate, :key) }
   let(:pages_domain_secure_missing_chain_params) {build(:pages_domain, :with_missing_chain, project: project).slice(:certificate) }
 
   let(:route) { "/projects/#{project.id}/pages/domains" }

spec/services/projects/update_service_spec.rb

@@ -242,6 +242,27 @@ describe Projects::UpdateService, '#execute' do
         })
       end
     end
+
+    context 'when updating #pages_https_only', :https_pages_enabled do
+      subject(:call_service) do
+        update_project(project, admin, pages_https_only: false)
+      end
+
+      it 'updates the attribute' do
+        expect { call_service }
+          .to change { project.pages_https_only? }
+          .to(false)
+      end
+
+      it 'calls Projects::UpdatePagesConfigurationService' do
+        expect(Projects::UpdatePagesConfigurationService)
+          .to receive(:new)
+          .with(project)
+          .and_call_original
+
+        call_service
+      end
+    end
   end
 
   describe '#run_auto_devops_pipeline?' do

spec/spec_helper.rb

@@ -219,6 +219,22 @@ RSpec.configure do |config|
       Ability.allowed?(*args)
     end
   end
+
+  config.before(:each, :http_pages_enabled) do |_|
+    allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80'])
+  end
+
+  config.before(:each, :https_pages_enabled) do |_|
+    allow(Gitlab.config.pages).to receive(:external_https).and_return(['1.1.1.1:443'])
+  end
+
+  config.before(:each, :http_pages_disabled) do |_|
+    allow(Gitlab.config.pages).to receive(:external_http).and_return(false)
+  end
+
+  config.before(:each, :https_pages_disabled) do |_|
+    allow(Gitlab.config.pages).to receive(:external_https).and_return(false)
+  end
 end
 
 # add simpler way to match asset paths containing digest strings