Merge branch 'ce-to-ee-2018-07-30' into 'master'

CE upstream - 2018-07-30 21:22 UTC

Closes gitaly#768, #7038, and gitlab-ce#49725

See merge request gitlab-org/gitlab-ee!6714

GITLAB_WORKHORSE_VERSION

-5.0.0
+5.1.0

Gemfile

@@ -318,7 +318,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~> 0.9.3'
+  gem 'prometheus-client-mmap', '~> 0.9.4'
   gem 'raindrops', '~> 0.18'
 end
 

Gemfile.lock

@@ -664,7 +664,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.9.3)
+    prometheus-client-mmap (0.9.4)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -1163,7 +1163,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.9.3)
+  prometheus-client-mmap (~> 0.9.4)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

Gemfile.rails5.lock

@@ -668,7 +668,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.9.3)
+    prometheus-client-mmap (0.9.4)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -1173,7 +1173,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.9.3)
+  prometheus-client-mmap (~> 0.9.4)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

app/assets/javascripts/diffs/components/diff_file_header.vue

@@ -50,7 +50,7 @@ export default {
     };
   },
   computed: {
-    ...mapGetters('diffs', ['diffHasExpandedDiscussions']),
+    ...mapGetters('diffs', ['diffHasExpandedDiscussions', 'diffHasDiscussions']),
     hasExpandedDiscussions() {
       return this.diffHasExpandedDiscussions(this.diffFile);
     },
@@ -108,6 +108,9 @@ export default {
         false,
       );
     },
+    gfmCopyText() {
+      return `\`${this.diffFile.filePath}\``;
+    },
   },
   methods: {
     ...mapActions('diffs', ['toggleFileDiscussions']),
@@ -191,6 +194,7 @@ export default {
       <clipboard-button
         :title="__('Copy file path to clipboard')"
         :text="diffFile.filePath"
+        :gfm="gfmCopyText"
         css-class="btn-default btn-transparent btn-clipboard"
       />
 
@@ -217,6 +221,7 @@ export default {
         v-if="diffFile.blob && diffFile.blob.readableText"
       >
         <button
+          :disabled="!diffHasDiscussions(diffFile)"
           :class="{ active: hasExpandedDiscussions }"
           :title="s__('MergeRequests|Toggle comments for this file')"
           class="js-btn-vue-toggle-comments btn"

app/assets/javascripts/diffs/store/getters.js

@@ -47,6 +47,14 @@ export const diffHasExpandedDiscussions = (state, getters) => diff => {
   );
 };
 
+/**
+ * Checks if the diff has any discussion
+ * @param {Boolean} diff
+ * @returns {Boolean}
+ */
+export const diffHasDiscussions = (state, getters) => diff =>
+  getters.getDiffFileDiscussions(diff).length > 0;
+
 /**
  * Returns an array with the discussions of the given diff
  * @param {Object} diff

app/assets/javascripts/ide/components/ide.vue

 <script>
 import Mousetrap from 'mousetrap';
 import { mapActions, mapState, mapGetters } from 'vuex';
+import { __ } from '~/locale';
 import NewModal from './new_dropdown/modal.vue';
 import IdeSidebar from './ide_side_bar.vue';
 import RepoTabs from './repo_tabs.vue';
@@ -25,7 +26,6 @@ export default {
   },
   computed: {
     ...mapState([
-      'changedFiles',
       'openFiles',
       'viewer',
       'currentMergeRequestId',
@@ -34,18 +34,10 @@ export default {
       'currentProjectId',
       'errorMessage',
     ]),
-    ...mapGetters(['activeFile', 'hasChanges']),
+    ...mapGetters(['activeFile', 'hasChanges', 'someUncommitedChanges']),
   },
   mounted() {
-    const returnValue = 'Are you sure you want to lose unsaved changes?';
-    window.onbeforeunload = e => {
-      if (!this.changedFiles.length) return undefined;
-
-      Object.assign(e, {
-        returnValue,
-      });
-      return returnValue;
-    };
+    window.onbeforeunload = e => this.onBeforeUnload(e);
 
     Mousetrap.bind(['t', 'command+p', 'ctrl+p'], e => {
       if (e.preventDefault) {
@@ -59,6 +51,16 @@ export default {
   },
   methods: {
     ...mapActions(['toggleFileFinder']),
+    onBeforeUnload(e = {}) {
+      const returnValue = __('Are you sure you want to lose unsaved changes?');
+
+      if (!this.someUncommitedChanges) return undefined;
+
+      Object.assign(e, {
+        returnValue,
+      });
+      return returnValue;
+    },
     mousetrapStopCallback(e, el, combo) {
       if (
         (combo === 't' && el.classList.contains('dropdown-input-field')) ||

app/assets/javascripts/lib/utils/http_status.js

@@ -2,11 +2,34 @@
  * exports HTTP status codes
  */
 
-export default {
+const httpStatusCodes = {
   ABORTED: 0,
-  NO_CONTENT: 204,
   OK: 200,
+  CREATED: 201,
+  ACCEPTED: 202,
+  NON_AUTHORITATIVE_INFORMATION: 203,
+  NO_CONTENT: 204,
+  RESET_CONTENT: 205,
+  PARTIAL_CONTENT: 206,
+  MULTI_STATUS: 207,
+  ALREADY_REPORTED: 208,
+  IM_USED: 226,
   MULTIPLE_CHOICES: 300,
   BAD_REQUEST: 400,
   NOT_FOUND: 404,
 };
+
+export const successCodes = [
+  httpStatusCodes.OK,
+  httpStatusCodes.CREATED,
+  httpStatusCodes.ACCEPTED,
+  httpStatusCodes.NON_AUTHORITATIVE_INFORMATION,
+  httpStatusCodes.NO_CONTENT,
+  httpStatusCodes.RESET_CONTENT,
+  httpStatusCodes.PARTIAL_CONTENT,
+  httpStatusCodes.MULTI_STATUS,
+  httpStatusCodes.ALREADY_REPORTED,
+  httpStatusCodes.IM_USED,
+];
+
+export default httpStatusCodes;

app/assets/javascripts/lib/utils/poll.js

-import httpStatusCodes from './http_status';
+import httpStatusCodes, { successCodes } from './http_status';
 import { normalizeHeaders } from './common_utils';
 
 /**
@@ -62,7 +62,7 @@ export default class Poll {
   checkConditions(response) {
     const headers = normalizeHeaders(response.headers);
     const pollInterval = parseInt(headers[this.intervalHeader], 10);
-    if (pollInterval > 0 && response.status === httpStatusCodes.OK && this.canPoll) {
+    if (pollInterval > 0 && successCodes.indexOf(response.status) !== -1 && this.canPoll) {
       clearTimeout(this.timeoutID);
       this.timeoutID = setTimeout(() => {
         this.makeRequest();

app/assets/javascripts/pipelines/components/graph/dropdown_job_component.vue

@@ -14,7 +14,7 @@ import tooltip from '../../../vue_shared/directives/tooltip';
  *   "id": 4256,
  *   "name": "test",
  *   "status": {
- *     "icon": "icon_status_success",
+ *     "icon": "status_success",
  *     "text": "passed",
  *     "label": "passed",
  *     "group": "success",

app/assets/javascripts/pipelines/components/graph/job_component.vue

@@ -13,7 +13,7 @@ import tooltip from '../../../vue_shared/directives/tooltip';
  *   "id": 4256,
  *   "name": "test",
  *   "status": {
- *     "icon": "icon_status_success",
+ *     "icon": "status_success",
  *     "text": "passed",
  *     "label": "passed",
  *     "group": "success",

app/assets/javascripts/vue_shared/components/clipboard_button.vue

@@ -31,6 +31,11 @@ export default {
       type: String,
       required: true,
     },
+    gfm: {
+      type: String,
+      required: false,
+      default: null,
+    },
     title: {
       type: String,
       required: true,
@@ -51,6 +56,14 @@ export default {
       default: 'btn-default',
     },
   },
+  computed: {
+    clipboardText() {
+      if (this.gfm !== null) {
+        return JSON.stringify({ text: this.text, gfm: this.gfm });
+      }
+      return this.text;
+    },
+  },
 };
 </script>
 
@@ -59,7 +72,7 @@ export default {
     v-tooltip
     :class="cssClass"
     :title="title"
-    :data-clipboard-text="text"
+    :data-clipboard-text="clipboardText"
     :data-container="tooltipContainer"
     :data-placement="tooltipPlacement"
     type="button"

app/controllers/admin/jobs_controller.rb

@@ -2,7 +2,7 @@ class Admin::JobsController < Admin::ApplicationController
   def index
     @scope = params[:scope]
     @all_builds = Ci::Build
-    @builds = @all_builds.order('created_at DESC')
+    @builds = @all_builds.order('id DESC')
     @builds =
       case @scope
       when 'pending'

app/controllers/application_controller.rb

@@ -405,7 +405,7 @@ class ApplicationController < ActionController::Base
       # actually stored in the session and a token is needed
       # for every request. If you want the token to work as a
       # sign in token, you can simply remove store: false.
-      sign_in user, store: false
+      sign_in(user, store: false, message: :sessionless_sign_in)
     end
   end
 

app/controllers/concerns/authenticates_with_two_factor.rb

@@ -60,7 +60,7 @@ module AuthenticatesWithTwoFactor
 
       remember_me(user) if user_params[:remember_me] == '1'
       user.save!
-      sign_in(user)
+      sign_in(user, message: :two_factor_authenticated)
     else
       user.increment_failed_attempts!
       Gitlab::AppLogger.info("Failed Login: user=#{user.username} ip=#{request.remote_ip} method=OTP")
@@ -77,7 +77,7 @@ module AuthenticatesWithTwoFactor
       session.delete(:challenge)
 
       remember_me(user) if user_params[:remember_me] == '1'
-      sign_in(user)
+      sign_in(user, message: :two_factor_authenticated)
     else
       user.increment_failed_attempts!
       Gitlab::AppLogger.info("Failed Login: user=#{user.username} ip=#{request.remote_ip} method=U2F")

app/controllers/projects/wikis_controller.rb

 class Projects::WikisController < Projects::ApplicationController
   include PreviewMarkdown
+  include Gitlab::Utils::StrongMemoize
 
   before_action :authorize_read_wiki!
   before_action :authorize_create_wiki!, only: [:edit, :create, :history]
   before_action :authorize_admin_wiki!, only: :destroy
   before_action :load_project_wiki
+  before_action :load_page, only: [:show, :edit, :update, :history, :destroy]
+  before_action :valid_encoding?, only: [:show, :edit, :update], if: :load_page
+  before_action only: [:edit, :update], unless: :valid_encoding? do
+    redirect_to(project_wiki_path(@project, @page))
+  end
 
   def pages
     @wiki_pages = Kaminari.paginate_array(@project_wiki.pages).page(params[:page])
@@ -12,11 +18,11 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def show
-    @page = @project_wiki.find_page(params[:id], params[:version_id])
-
     view_param = @project_wiki.empty? ? params[:view] : 'create'
 
     if @page
+      set_encoding_error unless valid_encoding?
+
       render 'show'
     elsif file = @project_wiki.find_file(params[:id], params[:version_id])
       response.headers['Content-Security-Policy'] = "default-src 'none'"
@@ -38,13 +44,11 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def edit
-    @page = @project_wiki.find_page(params[:id])
   end
 
   def update
     return render('empty') unless can?(current_user, :create_wiki, @project)
 
-    @page = @project_wiki.find_page(params[:id])
     @page = WikiPages::UpdateService.new(@project, current_user, wiki_params).execute(@page)
 
     if @page.valid?
@@ -79,8 +83,6 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def history
-    @page = @project_wiki.find_page(params[:id])
-
     if @page
       @page_versions = Kaminari.paginate_array(@page.versions(page: params[:page].to_i),
                                                total_count: @page.count_versions)
@@ -94,8 +96,6 @@ class Projects::WikisController < Projects::ApplicationController
   end
 
   def destroy
-    @page = @project_wiki.find_page(params[:id])
-
     WikiPages::DestroyService.new(@project, current_user).execute(@page)
 
     redirect_to project_wiki_path(@project, :home),
@@ -141,4 +141,25 @@ class Projects::WikisController < Projects::ApplicationController
       page.update_attributes(args) # rubocop:disable Rails/ActiveRecordAliases
     end
   end
+
+  def load_page
+    @page ||= @project_wiki.find_page(*page_params)
+  end
+
+  def page_params
+    keys = [:id]
+    keys << :version_id if params[:action] == 'show'
+
+    params.values_at(*keys)
+  end
+
+  def valid_encoding?
+    strong_memoize(:valid_encoding) do
+      @page.content.encoding == Encoding::UTF_8
+    end
+  end
+
+  def set_encoding_error
+    flash.now[:notice] = "The content of this page is not encoded in UTF-8. Edits can only be made via the Git repository."
+  end
 end

app/controllers/sessions_controller.rb

@@ -90,6 +90,14 @@ class SessionsController < Devise::SessionsController
     ).increment
   end
 
+  ##
+  # We do have some duplication between lib/gitlab/auth/activity.rb here, but
+  # leaving this method here because of backwards compatibility.
+  #
+  def login_counter
+    @login_counter ||= Gitlab::Metrics.counter(:user_session_logins_total, 'User sign in count')
+  end
+
   def log_failed_login
     Gitlab::AppLogger.info("Failed Login: username=#{user_params[:login]} ip=#{request.remote_ip}")
   end
@@ -98,10 +106,6 @@ class SessionsController < Devise::SessionsController
     (options = env["warden.options"]) && options[:action] == "unauthenticated"
   end
 
-  def login_counter
-    @login_counter ||= Gitlab::Metrics.counter(:user_session_logins_total, 'User sign in count')
-  end
-
   # Handle an "initial setup" state, where there's only one user, it's an admin,
   # and they require a password change.
   def check_initial_setup

app/models/application_setting.rb

@@ -228,8 +228,8 @@ class ApplicationSetting < ActiveRecord::Base
   def self.defaults
     {
       after_sign_up_text: nil,
-      akismet_enabled: false,
       allow_local_requests_from_hooks_and_services: false,
+      akismet_enabled: false,
       authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand
       container_registry_token_expire_delay: 5,
       default_artifacts_expire_in: '30 days',

app/models/ci/build.rb

@@ -26,9 +26,10 @@ module Ci
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
 
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
-    has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
-    has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
-    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+
+    Ci::JobArtifact.file_types.each do |key, value|
+      has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    end
 
     has_one :metadata, class_name: 'Ci::BuildMetadata'
     has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
@@ -390,6 +391,10 @@ module Ci
       trace.exist?
     end
 
+    def has_test_reports?
+      job_artifacts.test_reports.any?
+    end
+
     def has_old_trace?
       old_trace.present?
     end
@@ -457,16 +462,22 @@ module Ci
       save
     end
 
+    def erase_test_reports!
+      # TODO: Use fast_destroy_all in the context of https://gitlab.com/gitlab-org/gitlab-ce/issues/35240
+      job_artifacts_junit&.destroy
+    end
+
     def erase(opts = {})
       return false unless erasable?
 
       erase_artifacts!
+      erase_test_reports!
       erase_trace!
       update_erased!(opts[:erased_by])
     end
 
     def erasable?
-      complete? && (artifacts? || has_trace?)
+      complete? && (artifacts? || has_test_reports? || has_trace?)
     end
 
     def erased?
@@ -543,10 +554,6 @@ module Ci
       Gitlab::Ci::Build::Image.from_services(self)
     end
 
-    def artifacts
-      [options[:artifacts]]
-    end
-
     def cache
       cache = options[:cache]
 

app/models/ci/build_runner_session.rb

@@ -17,7 +17,7 @@ module Ci
       {
         subprotocols: ['terminal.gitlab.com'].freeze,
         url: "#{url}/exec".sub("https://", "wss://"),
-        headers: { Authorization: authorization.presence }.compact,
+        headers: { Authorization: [authorization.presence] }.compact,
         ca_pem: certificate.presence
       }
     end

app/models/ci/job_artifact.rb

@@ -6,11 +6,17 @@ module Ci
     include ObjectStorage::BackgroundMove
     extend Gitlab::Ci::Model
 
+    TEST_REPORT_FILE_TYPES = %w[junit].freeze
+    DEFAULT_FILE_NAMES = { junit: 'junit.xml' }.freeze
+    TYPE_AND_FORMAT_PAIRS = { archive: :zip, metadata: :gzip, trace: :raw, junit: :gzip }.freeze
+
     belongs_to :project
     belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
 
     mount_uploader :file, JobArtifactUploader
 
+    validates :file_format, presence: true, unless: :trace?, on: :create
+    validate :valid_file_format?, unless: :trace?, on: :create
     before_save :set_size, if: :file_changed?
     after_save :update_project_statistics_after_save, if: :size_changed?
     after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed?
@@ -20,14 +26,33 @@ module Ci
     scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
     scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
 
+    scope :test_reports, -> do
+      types = self.file_types.select { |file_type| TEST_REPORT_FILE_TYPES.include?(file_type) }.values
+
+      where(file_type: types)
+    end
+
     delegate :exists?, :open, to: :file
 
     enum file_type: {
       archive: 1,
       metadata: 2,
-      trace: 3
+      trace: 3,
+      junit: 4
     }
 
+    enum file_format: {
+      raw: 1,
+      zip: 2,
+      gzip: 3
+    }
+
+    def valid_file_format?
+      unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
+        errors.add(:file_format, 'Invalid file format with specified file type')
+      end
+    end
+
     def update_file_store
       # The file.object_store is set during `uploader.store!`
       # which happens after object is inserted/updated

app/models/concerns/atomic_internal_id.rb

@@ -26,6 +26,10 @@ module AtomicInternalId
 
   module ClassMethods
     def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName
+      # We require init here to retain the ability to recalculate in the absence of a
+      # InternaLId record (we may delete records in `internal_ids` for example).
+      raise "has_internal_id requires a init block, none given." unless init
+
       before_validation :"ensure_#{scope}_#{column}!", on: :create
       validates column, presence: presence
 

app/models/milestone.rb

@@ -153,6 +153,10 @@ class Milestone < ActiveRecord::Base
       reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC'))
     when 'due_date_desc'
       reorder(Gitlab::Database.nulls_last_order('due_date', 'DESC'))
+    when 'name_asc'
+      reorder(Arel::Nodes::Ascending.new(arel_table[:title].lower))
+    when 'name_desc'
+      reorder(Arel::Nodes::Descending.new(arel_table[:title].lower))
     when 'start_date_asc'
       reorder(Gitlab::Database.nulls_last_order('start_date', 'ASC'))
     when 'start_date_desc'

app/models/project.rb

@@ -563,6 +563,10 @@ class Project < ActiveRecord::Base
     repository.commit_by(oid: oid)
   end
 
+  def commits_by(oids:)
+    repository.commits_by(oids: oids)
+  end
+
   # ref can't be HEAD, can only be branch/tag name or SHA
   def latest_successful_builds_for(ref = default_branch)
     latest_pipeline = pipelines.latest_successful_for(ref)

app/models/project_statistics.rb

@@ -5,7 +5,7 @@ class ProjectStatistics < ActiveRecord::Base
   before_save :update_storage_size
 
   COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze
-  INCREMENTABLE_COLUMNS = [:build_artifacts_size].freeze
+  INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size] }.freeze
 
   def shared_runners_minutes
     shared_runners_seconds.to_i / 60
@@ -42,11 +42,28 @@ class ProjectStatistics < ActiveRecord::Base
     self.storage_size = repository_size + lfs_objects_size + build_artifacts_size
   end
 
+  # Since this incremental update method does not call update_storage_size above,
+  # we have to update the storage_size here as additional column.
+  # Additional columns are updated depending on key => [columns], which allows
+  # to update statistics which are and also those which aren't included in storage_size
+  # or any other additional summary column in the future.
   def self.increment_statistic(project_id, key, amount)
-    raise ArgumentError, "Cannot increment attribute: #{key}" unless key.in?(INCREMENTABLE_COLUMNS)
+    raise ArgumentError, "Cannot increment attribute: #{key}" unless INCREMENTABLE_COLUMNS.key?(key)
     return if amount == 0
 
     where(project_id: project_id)
-      .update_all(["#{key} = COALESCE(#{key}, 0) + (?)", amount])
+      .columns_to_increment(key, amount)
+  end
+
+  def self.columns_to_increment(key, amount)
+    updates = ["#{key} = COALESCE(#{key}, 0) + (#{amount})"]
+
+    if (additional = INCREMENTABLE_COLUMNS[key])
+      additional.each do |column|
+        updates << "#{column} = COALESCE(#{column}, 0) + (#{amount})"
+      end
+    end
+
+    update_all(updates.join(', '))
   end
 end

app/models/repository.rb

@@ -317,6 +317,8 @@ class Repository
   # types - An Array of file types (e.g. `:readme`) used to refresh extra
   #         caches.
   def refresh_method_caches(types)
+    return if types.empty?
+
     to_refresh = []
 
     types.each do |type|

app/models/user.rb

@@ -257,6 +257,7 @@ class User < ActiveRecord::Base
   scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) }
   scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) }
   scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) }
+  scope :confirmed, -> { where.not(confirmed_at: nil) }
 
   def self.with_two_factor_indistinct
     joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id")
@@ -302,14 +303,17 @@ class User < ActiveRecord::Base
     end
 
     # Find a User by their primary email or any associated secondary email
-    def find_by_any_email(email)
-      by_any_email(email).take
+    def find_by_any_email(email, confirmed: false)
+      by_any_email(email, confirmed: confirmed).take
     end
 
     # Returns a relation containing all the users for the given Email address
-    def by_any_email(email)
+    def by_any_email(email, confirmed: false)
       users = where(email: email)
+      users = users.confirmed if confirmed
+
       emails = joins(:emails).where(emails: { email: email })
+      emails = emails.confirmed if confirmed
       union = Gitlab::SQL::Union.new([users, emails])
 
       from("(#{union.to_sql}) #{table_name}")

app/presenters/ci/build_runner_presenter.rb

+module Ci
+  class BuildRunnerPresenter < SimpleDelegator
+    def artifacts
+      return unless options[:artifacts]
+
+      list = []
+      list << create_archive(options[:artifacts])
+      list << create_reports(options[:artifacts][:reports], expire_in: options[:artifacts][:expire_in])
+      list.flatten.compact
+    end
+
+    private
+
+    def create_archive(artifacts)
+      return unless artifacts[:untracked] || artifacts[:paths]
+
+      {
+        artifact_type: :archive,
+        artifact_format: :zip,
+        name: artifacts[:name],
+        untracked: artifacts[:untracked],
+        paths: artifacts[:paths],
+        when: artifacts[:when],
+        expire_in: artifacts[:expire_in]
+      }
+    end
+
+    def create_reports(reports, expire_in:)
+      return unless reports&.any?
+
+      reports.map do |k, v|
+        {
+          artifact_type: k.to_sym,
+          artifact_format: :gzip,
+          name: ::Ci::JobArtifact::DEFAULT_FILE_NAMES[k.to_sym],
+          paths: v,
+          when: 'always',
+          expire_in: expire_in
+        }
+      end
+    end
+  end
+end

app/services/git_push_service.rb

@@ -3,6 +3,7 @@
 class GitPushService < BaseService
   attr_accessor :push_data, :push_commits
   include Gitlab::Access
+  include Gitlab::Utils::StrongMemoize
 
   # The N most recent commits to process in a single push payload.
   PROCESS_COMMIT_LIMIT = 100
@@ -21,14 +22,14 @@ class GitPushService < BaseService
   #  6. Checks if the project's main language has changed
   #
   def execute
-    @project.repository.after_create if @project.empty_repo?
-    @project.repository.after_push_commit(branch_name)
+    project.repository.after_create if project.empty_repo?
+    project.repository.after_push_commit(branch_name)
 
     if push_remove_branch?
-      @project.repository.after_remove_branch
+      project.repository.after_remove_branch
       @push_commits = []
     elsif push_to_new_branch?
-      @project.repository.after_create_branch
+      project.repository.after_create_branch
 
       # Re-find the pushed commits.
       if default_branch?
@@ -38,14 +39,14 @@ class GitPushService < BaseService
         # Use the pushed commits that aren't reachable by the default branch
         # as a heuristic. This may include more commits than are actually pushed, but
         # that shouldn't matter because we check for existing cross-references later.
-        @push_commits = @project.repository.commits_between(@project.default_branch, params[:newrev])
+        @push_commits = project.repository.commits_between(project.default_branch, params[:newrev])
 
         # don't process commits for the initial push to the default branch
         process_commit_messages
       end
     elsif push_to_existing_branch?
       # Collect data for this git push
-      @push_commits = @project.repository.commits_between(params[:oldrev], params[:newrev])
+      @push_commits = project.repository.commits_between(params[:oldrev], params[:newrev])
 
       process_commit_messages
 
@@ -68,7 +69,7 @@ class GitPushService < BaseService
   end
 
   def update_gitattributes
-    @project.repository.copy_gitattributes(params[:ref])
+    project.repository.copy_gitattributes(params[:ref])
   end
 
   def update_caches
@@ -80,7 +81,7 @@ class GitPushService < BaseService
       else
         paths = Set.new
 
-        @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+        last_pushed_commits.each do |commit|
           commit.raw_deltas.each do |diff|
             paths << diff.new_path
           end
@@ -92,11 +93,11 @@ class GitPushService < BaseService
       types = []
     end
 
-    ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
+    ProjectCacheWorker.perform_async(project.id, types, [:commit_count, :repository_size])
   end
 
   def update_signatures
-    commit_shas = @push_commits.last(PROCESS_COMMIT_LIMIT).map(&:sha)
+    commit_shas = last_pushed_commits.map(&:sha)
 
     return if commit_shas.empty?
 
@@ -107,16 +108,14 @@ class GitPushService < BaseService
 
     commit_shas = Gitlab::Git::Commit.shas_with_signatures(project.repository, commit_shas)
 
-    commit_shas.each do |sha|
-      CreateGpgSignatureWorker.perform_async(sha, project.id)
-    end
+    CreateGpgSignatureWorker.perform_async(commit_shas, project.id)
   end
 
   # Schedules processing of commit messages.
   def process_commit_messages
     default = default_branch?
 
-    @push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+    last_pushed_commits.each do |commit|
       if commit.matches_cross_reference_regex?
         ProcessCommitWorker
           .perform_async(project.id, current_user.id, commit.to_hash, default)
@@ -127,10 +126,10 @@ class GitPushService < BaseService
   protected
 
   def update_remote_mirrors
-    return unless @project.has_remote_mirror?
+    return unless project.has_remote_mirror?
 
-    @project.mark_stuck_remote_mirrors_as_failed!
-    @project.update_remote_mirrors
+    project.mark_stuck_remote_mirrors_as_failed!
+    project.update_remote_mirrors
   end
 
   def execute_related_hooks
@@ -138,16 +137,16 @@ class GitPushService < BaseService
     # could cause the last commit of a merge request to change.
     #
     UpdateMergeRequestsWorker
-      .perform_async(@project.id, current_user.id, params[:oldrev], params[:newrev], params[:ref])
+      .perform_async(project.id, current_user.id, params[:oldrev], params[:newrev], params[:ref])
 
-    mirror_update = @project.mirror? && @project.repository.up_to_date_with_upstream?(branch_name)
+    mirror_update = project.mirror? && project.repository.up_to_date_with_upstream?(branch_name)
 
-    EventCreateService.new.push(@project, current_user, build_push_data)
-    Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute(:push, mirror_update: mirror_update)
+    EventCreateService.new.push(project, current_user, build_push_data)
+    Ci::CreatePipelineService.new(project, current_user, build_push_data).execute(:push, mirror_update: mirror_update)
 
     SystemHookPushWorker.perform_async(build_push_data.dup, :push_hooks)
-    @project.execute_hooks(build_push_data.dup, :push_hooks)
-    @project.execute_services(build_push_data.dup, :push_hooks)
+    project.execute_hooks(build_push_data.dup, :push_hooks)
+    project.execute_services(build_push_data.dup, :push_hooks)
 
     if push_remove_branch?
       AfterBranchDeleteService
@@ -157,52 +156,50 @@ class GitPushService < BaseService
   end
 
   def perform_housekeeping
-    housekeeping = Projects::HousekeepingService.new(@project)
+    housekeeping = Projects::HousekeepingService.new(project)
     housekeeping.increment!
     housekeeping.execute if housekeeping.needed?
   rescue Projects::HousekeepingService::LeaseTaken
   end
 
   def process_default_branch
-    @push_commits_count = project.repository.commit_count_for_ref(params[:ref])
-
-    offset = [@push_commits_count - PROCESS_COMMIT_LIMIT, 0].max
+    offset = [push_commits_count - PROCESS_COMMIT_LIMIT, 0].max
     @push_commits = project.repository.commits(params[:newrev], offset: offset, limit: PROCESS_COMMIT_LIMIT)
 
-    @project.after_create_default_branch
+    project.after_create_default_branch
   end
 
   def build_push_data
     @push_data ||= Gitlab::DataBuilder::Push.build(
-      @project,
+      project,
       current_user,
       params[:oldrev],
       params[:newrev],
       params[:ref],
       @push_commits,
-      commits_count: @push_commits_count)
+      commits_count: push_commits_count)
   end
 
   def push_to_existing_branch?
     # Return if this is not a push to a branch (e.g. new commits)
-    Gitlab::Git.branch_ref?(params[:ref]) && !Gitlab::Git.blank_ref?(params[:oldrev])
+    branch_ref? && !Gitlab::Git.blank_ref?(params[:oldrev])
   end
 
   def push_to_new_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) && Gitlab::Git.blank_ref?(params[:oldrev])
+    strong_memoize(:push_to_new_branch) do
+      branch_ref? && Gitlab::Git.blank_ref?(params[:oldrev])
+    end
   end
 
   def push_remove_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) && Gitlab::Git.blank_ref?(params[:newrev])
-  end
-
-  def push_to_branch?
-    Gitlab::Git.branch_ref?(params[:ref])
+    strong_memoize(:push_remove_branch) do
+      branch_ref? && Gitlab::Git.blank_ref?(params[:newrev])
+    end
   end
 
   def default_branch?
-    Gitlab::Git.branch_ref?(params[:ref]) &&
-      (Gitlab::Git.ref_name(params[:ref]) == project.default_branch || project.default_branch.nil?)
+    branch_ref? &&
+      (branch_name == project.default_branch || project.default_branch.nil?)
   end
 
   def commit_user(commit)
@@ -210,6 +207,24 @@ class GitPushService < BaseService
   end
 
   def branch_name
-    @branch_name ||= Gitlab::Git.ref_name(params[:ref])
+    strong_memoize(:branch_name) do
+      Gitlab::Git.ref_name(params[:ref])
+    end
+  end
+
+  def branch_ref?
+    strong_memoize(:branch_ref) do
+      Gitlab::Git.branch_ref?(params[:ref])
+    end
+  end
+
+  def push_commits_count
+    strong_memoize(:push_commits_count) do
+      project.repository.commit_count_for_ref(params[:ref])
+    end
+  end
+
+  def last_pushed_commits
+    @last_pushed_commits ||= @push_commits.last(PROCESS_COMMIT_LIMIT)
   end
 end

app/services/git_tag_push_service.rb

@@ -9,12 +9,12 @@ class GitTagPushService < BaseService
 
     @push_data = build_push_data
 
-    EventCreateService.new.push(project, current_user, @push_data)
-    Ci::CreatePipelineService.new(project, current_user, @push_data).execute(:push, mirror_update: params[:mirror_update])
+    EventCreateService.new.push(project, current_user, push_data)
+    Ci::CreatePipelineService.new(project, current_user, push_data).execute(:push, mirror_update: params[:mirror_update])
 
-    SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks)
-    project.execute_hooks(@push_data.dup, :tag_push_hooks)
-    project.execute_services(@push_data.dup, :tag_push_hooks)
+    SystemHooksService.new.execute_hooks(build_system_push_data, :tag_push_hooks)
+    project.execute_hooks(push_data.dup, :tag_push_hooks)
+    project.execute_services(push_data.dup, :tag_push_hooks)
 
     ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 

app/services/issues/update_service.rb

@@ -37,6 +37,8 @@ module Issues
       end
 
       if issue.previous_changes.include?('confidential')
+        # don't enqueue immediately to prevent todos removal in case of a mistake
+        TodosDestroyer::ConfidentialIssueWorker.perform_in(1.hour, issue.id) if issue.confidential?
         create_confidentiality_note(issue)
       end
 

app/services/members/destroy_service.rb

@@ -17,6 +17,8 @@ module Members
         notification_service.decline_access_request(member)
       end
 
+      enqeue_delete_todos(member)
+
       after_execute(member: member)
 
       member
@@ -24,6 +26,12 @@ module Members
 
     private
 
+    def enqeue_delete_todos(member)
+      type = member.is_a?(GroupMember) ? 'Group' : 'Project'
+      # don't enqueue immediately to prevent todos removal in case of a mistake
+      TodosDestroyer::EntityLeaveWorker.perform_in(1.hour, member.user_id, member.source_id, type)
+    end
+
     def can_destroy_member?(member)
       can?(current_user, destroy_member_permission(member), member)
     end

app/services/projects/update_service.rb

@@ -27,13 +27,7 @@ module Projects
       return validation_failed! if project.errors.any?
 
       if project.update(params.except(:default_branch))
-        if project.previous_changes.include?('path')
-          project.rename_repo
-        else
-          system_hook_service.execute_hooks_for(project, :update)
-        end
-
-        update_pages_config if changing_pages_https_only?
+        after_update
 
         success
       else
@@ -49,6 +43,30 @@ module Projects
 
     private
 
+    def after_update
+      todos_features_changes = %w(
+        issues_access_level
+        merge_requests_access_level
+        repository_access_level
+      )
+      project_changed_feature_keys = project.project_feature.previous_changes.keys
+
+      if project.previous_changes.include?(:visibility_level) && project.private?
+        # don't enqueue immediately to prevent todos removal in case of a mistake
+        TodosDestroyer::ProjectPrivateWorker.perform_in(1.hour, project.id)
+      elsif (project_changed_feature_keys & todos_features_changes).present?
+        TodosDestroyer::PrivateFeaturesWorker.perform_in(1.hour, project.id)
+      end
+
+      if project.previous_changes.include?('path')
+        project.rename_repo
+      else
+        system_hook_service.execute_hooks_for(project, :update)
+      end
+
+      update_pages_config if changing_pages_https_only?
+    end
+
     def validation_failed!
       model_errors = project.errors.full_messages.to_sentence
       error_message = model_errors.presence || 'Project could not be updated!'

app/services/todos/destroy/base_service.rb

+module Todos
+  module Destroy
+    class BaseService
+      def execute
+        return unless todos_to_remove?
+
+        without_authorized(todos).delete_all
+      end
+
+      private
+
+      def without_authorized(items)
+        items.where('user_id NOT IN (?)', authorized_users)
+      end
+
+      def authorized_users
+        ProjectAuthorization.select(:user_id).where(project_id: project_ids)
+      end
+
+      def todos
+        raise NotImplementedError
+      end
+
+      def project_ids
+        raise NotImplementedError
+      end
+
+      def todos_to_remove?
+        raise NotImplementedError
+      end
+    end
+  end
+end

app/services/todos/destroy/confidential_issue_service.rb

+module Todos
+  module Destroy
+    class ConfidentialIssueService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :issue
+
+      def initialize(issue_id)
+        @issue = Issue.find_by(id: issue_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        Todo.where(target: issue)
+          .where('user_id != ?', issue.author_id)
+          .where('user_id NOT IN (?)', issue.assignees.select(:id))
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        issue&.confidential?
+      end
+
+      override :project_ids
+      def project_ids
+        issue.project_id
+      end
+
+      override :authorized_users
+      def authorized_users
+        ProjectAuthorization.select(:user_id)
+          .where(project_id: project_ids)
+          .where('access_level >= ?', Gitlab::Access::REPORTER)
+      end
+    end
+  end
+end

app/services/todos/destroy/entity_leave_service.rb

+module Todos
+  module Destroy
+    class EntityLeaveService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :user_id, :entity
+
+      def initialize(user_id, entity_id, entity_type)
+        unless %w(Group Project).include?(entity_type)
+          raise ArgumentError.new("#{entity_type} is not an entity user can leave")
+        end
+
+        @user_id = user_id
+        @entity = entity_type.constantize.find_by(id: entity_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        if entity.private?
+          Todo.where(project_id: project_ids, user_id: user_id)
+        else
+          project_ids.each do |project_id|
+            TodosDestroyer::PrivateFeaturesWorker.perform_async(project_id, user_id)
+          end
+
+          Todo.where(
+            target_id: confidential_issues.select(:id), target_type: Issue, user_id: user_id
+          )
+        end
+      end
+
+      override :project_ids
+      def project_ids
+        case entity
+        when Project
+          [entity.id]
+        when Namespace
+          Project.select(:id).where(namespace_id: entity.self_and_descendants.select(:id))
+        end
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        # if an entity is provided we want to check always at least private features
+        !!entity
+      end
+
+      def confidential_issues
+        assigned_ids = IssueAssignee.select(:issue_id).where(user_id: user_id)
+
+        Issue.where(project_id: project_ids, confidential: true)
+          .where('author_id != ?', user_id)
+          .where('id NOT IN (?)', assigned_ids)
+      end
+    end
+  end
+end

app/services/todos/destroy/private_features_service.rb

+module Todos
+  module Destroy
+    class PrivateFeaturesService < ::Todos::Destroy::BaseService
+      attr_reader :project_ids, :user_id
+
+      def initialize(project_ids, user_id = nil)
+        @project_ids = project_ids
+        @user_id = user_id
+      end
+
+      def execute
+        ProjectFeature.where(project_id: project_ids).each do |project_features|
+          target_types = []
+          target_types << Issue if private?(project_features.issues_access_level)
+          target_types << MergeRequest if private?(project_features.merge_requests_access_level)
+          target_types << Commit if private?(project_features.repository_access_level)
+
+          next if target_types.empty?
+
+          remove_todos(project_features.project_id, target_types)
+        end
+      end
+
+      private
+
+      def private?(feature_level)
+        feature_level == ProjectFeature::PRIVATE
+      end
+
+      def remove_todos(project_id, target_types)
+        items = Todo.where(project_id: project_id)
+        items = items.where(user_id: user_id) if user_id
+
+        items.where('user_id NOT IN (?)', authorized_users)
+          .where(target_type: target_types)
+          .delete_all
+      end
+    end
+  end
+end

app/services/todos/destroy/project_private_service.rb

+module Todos
+  module Destroy
+    class ProjectPrivateService < ::Todos::Destroy::BaseService
+      extend ::Gitlab::Utils::Override
+
+      attr_reader :project
+
+      def initialize(project_id)
+        @project = Project.find_by(id: project_id)
+      end
+
+      private
+
+      override :todos
+      def todos
+        Todo.where(project_id: project_ids)
+      end
+
+      override :project_ids
+      def project_ids
+        project.id
+      end
+
+      override :todos_to_remove?
+      def todos_to_remove?
+        project&.private?
+      end
+    end
+  end
+end

app/views/projects/wikis/_main_links.html.haml

@@ -4,6 +4,6 @@
       = s_("Wiki|New page")
   = link_to project_wiki_history_path(@project, @page), class: "btn" do
     = s_("Wiki|Page history")
-  - if can?(current_user, :create_wiki, @project) && @page.latest?
+  - if can?(current_user, :create_wiki, @project) && @page.latest? && @valid_encoding
     = link_to project_wiki_edit_path(@project, @page), class: "btn js-wiki-edit" do
       = _("Edit")

app/views/shared/boards/components/_board.html.haml

@@ -36,7 +36,6 @@
           %span.issue-count-badge-count
             %icon.mr-1{ name: "issues" }
             {{ list.issuesSize }}
-
           = render_if_exists "shared/boards/components/list_weight"
 
         - if can?(current_user, :admin_list, current_board_parent)

app/workers/all_queues.yml

@@ -75,6 +75,11 @@
 - repository_check:repository_check_batch
 - repository_check:repository_check_single_repository
 
+- todos_destroyer:todos_destroyer_confidential_issue
+- todos_destroyer:todos_destroyer_entity_leave
+- todos_destroyer:todos_destroyer_project_private
+- todos_destroyer:todos_destroyer_private_features
+
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
 

app/workers/concerns/todos_destroyer_queue.rb

+# frozen_string_literal: true
+
+##
+# Concern for setting Sidekiq settings for the various Todos Destroyers.
+#
+module TodosDestroyerQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :todos_destroyer
+  end
+end

app/workers/create_gpg_signature_worker.rb

@@ -3,15 +3,23 @@
 class CreateGpgSignatureWorker
   include ApplicationWorker
 
-  def perform(commit_sha, project_id)
+  def perform(commit_shas, project_id)
+    return if commit_shas.empty?
+
     project = Project.find_by(id: project_id)
     return unless project
 
-    commit = project.commit(commit_sha)
+    commits = project.commits_by(oids: commit_shas)
 
-    return unless commit
+    return if commits.empty?
 
     # This calculates and caches the signature in the database
-    Gitlab::Gpg::Commit.new(commit).signature
+    commits.each do |commit|
+      begin
+        Gitlab::Gpg::Commit.new(commit).signature
+      rescue => e
+        Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}")
+      end
+    end
   end
 end

app/workers/todos_destroyer/confidential_issue_worker.rb

+module TodosDestroyer
+  class ConfidentialIssueWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(issue_id)
+      ::Todos::Destroy::ConfidentialIssueService.new(issue_id).execute
+    end
+  end
+end

app/workers/todos_destroyer/entity_leave_worker.rb

+module TodosDestroyer
+  class EntityLeaveWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(user_id, entity_id, entity_type)
+      ::Todos::Destroy::EntityLeaveService.new(user_id, entity_id, entity_type).execute
+    end
+  end
+end

app/workers/todos_destroyer/private_features_worker.rb

+module TodosDestroyer
+  class PrivateFeaturesWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(project_id, user_id = nil)
+      ::Todos::Destroy::PrivateFeaturesService.new(project_id, user_id).execute
+    end
+  end
+end

app/workers/todos_destroyer/project_private_worker.rb

+module TodosDestroyer
+  class ProjectPrivateWorker
+    include ApplicationWorker
+    include TodosDestroyerQueue
+
+    def perform(project_id)
+      ::Todos::Destroy::ProjectPrivateService.new(project_id).execute
+    end
+  end
+end

changelogs/unreleased/25990-interactive-web-terminals-authorization.yml

+---
+title: Fix authorization for interactive web terminals
+merge_request: 20811
+author:
+type: fixed

changelogs/unreleased/32821-better-error-message-add-invalid-user-to-project.yml

+---
+title: Improve error message when adding invalid user to a project
+merge_request: 20885
+author: Jacopo Beschi @jacopo-beschi
+type: added

changelogs/unreleased/47728-mr-api-documentation-changes.yml

+---
+title: Remove changes_count from MR API documentation where necessary
+merge_request: 19745
+author: Jan Beckmann
+type: fixed

changelogs/unreleased/48823-copy-gfm.yml

+---
+title: Resolve Copy diff file path as GFM is broken
+merge_request: 20725
+author:
+type: fixed

changelogs/unreleased/49161-disable-toggle-comments.yml

+---
+title: Disables toggle comments button if diff has no discussions
+merge_request:
+author:
+type: other

changelogs/unreleased/49701-sorting-by-name-on-milestones-page-error.yml

+---
+title: Fix sorting by name on milestones page
+merge_request: 20881
+author:
+type: fixed

changelogs/unreleased/49747-update-poll-2xx.yml

+---
+title: Changes poll.js to keep polling on any 2xx http status code
+merge_request: 20904
+author:
+type: other

changelogs/unreleased/artifact-format-v2.yml

+---
+title: Extend gitlab-ci.yml to request junit.xml test reports
+merge_request: 20390
+author:
+type: added

changelogs/unreleased/feature-gb-login-activity-metrics.yml

+---
+title: Add more comprehensive metrics tracking authentication activity
+merge_request: 20668
+author:
+type: added

changelogs/unreleased/fix-storage-size-for-artifacts-change.yml

+---
+title: Update total storage size when changing size of artifacts
+merge_request: 20697
+author: Peter Marko
+type: fixed

changelogs/unreleased/fj-37736-improve-performance-post-receive-create-gpg-siganture-worker.yml

+---
+title: Performing Commit GPG signature calculation in bulk
+merge_request: 20870
+author:
+type: performance

changelogs/unreleased/fj-49512-fix-gitlab-git-pages-encoding.yml

+---
+title: Prevent editing and updating wiki pages with non UTF-8 encoding via web interface
+merge_request: 20906
+author:
+type: fixed

changelogs/unreleased/ide-warn-staged-files.yml

+---
+title: Warn user when reload IDE with staged changes
+merge_request: 20857
+author:
+type: added

changelogs/unreleased/rails5-gpg-permit-concurrent.yml

+---
+title: Permit concurrent loads in gpg keychain mutex
+merge_request: 20894
+author: Jasper Maes
+type: fixed

changelogs/unreleased/sh-fix-admin-jobs-controller-timing-out.yml

+---
+title: Fix /admin/jobs failing to load due to statement timeout
+merge_request: 20909
+author:
+type: performance

changelogs/unreleased/sh-simplify-liveness-check.yml

+---
+title: Add /-/health basic health check endpoint
+merge_request: 20456
+author:
+type: added

changelogs/unreleased/sh-support-users-find-by-confirmed-emails.yml

+---
+title: Add support for searching users by confirmed e-mails
+merge_request: 20893
+author:
+type: other

changelogs/unreleased/todos-visibility-change.yml

+---
+title: Delete todos when user loses access to read the target
+merge_request: 20665
+author:
+type: other

config/application.rb

@@ -182,6 +182,10 @@ module Gitlab
 
     config.action_view.sanitized_allowed_protocols = %w(smb)
 
+    # This middleware needs to precede ActiveRecord::QueryCache and other middlewares that
+    # connect to the database.
+    config.middleware.insert_after "Rails::Rack::Logger", "Gitlab::Middleware::BasicHealthCheck"
+
     config.middleware.insert_after Warden::Manager, Rack::Attack
 
     # Allow access to GitLab API from other domains

config/initializers/rbtrace.rb

-# frozen_string_literal: true
-
-require 'rbtrace' if ENV['ENABLE_RBTRACE']

config/initializers/sidekiq.rb

@@ -8,6 +8,8 @@ Sidekiq.default_worker_options = { retry: 3 }
 enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
 
 Sidekiq.configure_server do |config|
+  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+
   config.redis = queues_config_hash
 
   config.server_middleware do |chain|

config/initializers/warden.rb

 Rails.application.configure do |config|
   Warden::Manager.after_set_user(scope: :user) do |user, auth, opts|
     Gitlab::Auth::UniqueIpsLimiter.limit_user!(user)
-  end
 
-  Warden::Manager.before_failure(scope: :user) do |env, opts|
-    Gitlab::Auth::BlockedUserTracker.log_if_user_blocked(env)
+    activity = Gitlab::Auth::Activity.new(user, opts)
+
+    case opts[:event]
+    when :authentication
+      activity.user_authenticated!
+    when :set_user
+      activity.user_authenticated!
+      activity.user_session_override!
+    when :fetch # rubocop:disable Lint/EmptyWhen
+      # We ignore session fetch events
+    else
+      activity.user_session_override!
+    end
   end
 
   Warden::Manager.after_authentication(scope: :user) do |user, auth, opts|
@@ -15,7 +25,17 @@ Rails.application.configure do |config|
     ActiveSession.set(user, auth.request)
   end
 
-  Warden::Manager.before_logout(scope: :user) do |user, auth, opts|
-    ActiveSession.destroy(user || auth.user, auth.request.session.id)
+  Warden::Manager.before_failure(scope: :user) do |env, opts|
+    tracker = Gitlab::Auth::BlockedUserTracker.new(env)
+    tracker.log_blocked_user_activity! if tracker.user_blocked?
+
+    Gitlab::Auth::Activity.new(tracker.user, opts).user_authentication_failed!
+  end
+
+  Warden::Manager.before_logout(scope: :user) do |user_warden, auth, opts|
+    user = user_warden || auth.user
+
+    ActiveSession.destroy(user, auth.request.session.id)
+    Gitlab::Auth::Activity.new(user, opts).user_session_destroyed!
   end
 end

config/routes.rb

@@ -61,6 +61,7 @@ Rails.application.routes.draw do
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
 
   scope path: '-' do
+    # '/-/health' implemented by BasicHealthMiddleware
     get 'liveness' => 'health#liveness'
     get 'readiness' => 'health#readiness'
     post 'storage_check' => 'health#storage_check'

config/sidekiq_queues.yml

@@ -45,6 +45,7 @@
   - [github_import_advance_stage, 1]
   - [project_service, 1]
   - [delete_user, 1]
+  - [todos_destroyer, 1]
   - [delete_merged_branches, 1]
   - [authorized_projects, 1]
   - [expire_build_instance_artifacts, 1]

config/unicorn.rb.example

@@ -124,6 +124,10 @@ before_fork do |server, worker|
 end
 
 after_fork do |server, worker|
+  # Unicorn clears out signals before it forks, so rbtrace won't work
+  # unless it is enabled after the fork.
+  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)

db/migrate/20180705160945_add_file_format_to_ci_job_artifacts.rb

+class AddFileFormatToCiJobArtifacts < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :ci_job_artifacts, :file_format, :integer, limit: 2
+  end
+end

db/schema.rb

@@ -482,6 +482,7 @@ ActiveRecord::Schema.define(version: 20180722103201) do
     t.string "file"
     t.integer "file_store"
     t.binary "file_sha256"
+    t.integer "file_format", limit: 2
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree

doc/README.md

@@ -201,7 +201,8 @@ instant how code changes impact your production environment.
 - [Prometheus metrics](user/project/integrations/prometheus_library/metrics.md): Let Prometheus collect metrics from various services, like Kubernetes, NGINX, NGINX ingress controller, HAProxy, and Amazon Cloud Watch.
 - [GitLab Performance Monitoring](administration/monitoring/performance/index.md): Use InfluxDB and Grafana to monitor the performance of your GitLab instance (will be eventually replaced by Prometheus).
 - [Health check](user/admin_area/monitoring/health_check.md): GitLab provides liveness and readiness probes to indicate service health and reachability to required services.
-- [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have.
+- [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an
+  [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have.
 
 ## Getting started with GitLab
 

doc/administration/high_availability/nfs.md

@@ -39,19 +39,11 @@ Our support team will not be able to assist on performance issues related to
 file system access.
 
 Customers and users have reported that AWS EFS does not perform well for GitLab's
-use-case. There are several issues that can cause problems. For these reasons
-GitLab does not recommend using EFS with GitLab.
-
-- EFS bases allowed IOPS on volume size. The larger the volume, the more IOPS
-  are allocated. For smaller volumes, users may experience decent performance
-  for a period of time due to 'Burst Credits'. Over a period of weeks to months
-  credits may run out and performance will bottom out.
-- For larger volumes, allocated IOPS may not be the problem. Workloads where
-  many small files are written in a serialized manner are not well-suited for EFS.
-  EBS with an NFS server on top will perform much better.
-
-In addition, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`)
-because this will also affect performance. We recommend that the log files be
+use-case. Workloads where many small files are written in a serialized manner, like `git`,
+are not well-suited for EFS. EBS with an NFS server on top will perform much better.
+
+If you do choose to use EFS, avoid storing GitLab log files (e.g. those in `/var/log/gitlab`)
+there because this will also affect performance. We recommend that the log files be
 stored on a local volume.
 
 For more details on another person's experience with EFS, see

doc/administration/troubleshooting/debug.md

@@ -77,7 +77,12 @@ and more. However, this is not enabled by default. To enable it, define the
 gitlab_rails['env'] = {"ENABLE_RBTRACE" => "1"}
 ```
 
-Then reconfigure the system and restart Unicorn and Sidekiq.
+Then reconfigure the system and restart Unicorn and Sidekiq. To run this
+in Omnibus, run as root:
+
+```ruby
+/opt/gitlab/embedded/bin/ruby /opt/gitlab/embedded/bin/rbtrace
+```
 
 ## Common Problems
 

doc/api/merge_requests.md

@@ -15,11 +15,6 @@ given state (`opened`, `closed`, `locked`, or `merged`) or all of them (`all`).
 The pagination parameters `page` and `per_page` can be used to
 restrict the list of merge requests.
 
-**Note**: the `changes_count` value in the response is a string, not an
-integer. This is because when an MR has too many changes to display and store,
-it will be capped at 1,000. In that case, the API will return the string
-`"1000+"` for the changes count.
-
 ```
 GET /merge_requests
 GET /merge_requests?state=opened
@@ -104,7 +99,6 @@ Parameters:
     "sha": "8888888888888888888888888888888888888888",
     "merge_commit_sha": null,
     "user_notes_count": 1,
-    "changes_count": "1",
     "should_remove_source_branch": true,
     "force_remove_source_branch": false,
     "squash": false,
@@ -144,10 +138,6 @@ will be the same. In the case of a merge request from a fork,
 `target_project_id` and `project_id` will be the same and
 `source_project_id` will be the fork project's ID.
 
-**Note**: the `changes_count` value in the response is a string, not an
-integer. This is because when an MR has too many changes to display and store,
-it will be capped at 1,000. In that case, the API will return the string
-`"1000+"` for the changes count.
 
 Parameters:
 
@@ -224,7 +214,6 @@ Parameters:
     "sha": "8888888888888888888888888888888888888888",
     "merge_commit_sha": null,
     "user_notes_count": 1,
-    "changes_count": "1",
     "should_remove_source_branch": true,
     "force_remove_source_branch": false,
     "squash": false,
@@ -332,7 +321,6 @@ Parameters:
     "sha": "8888888888888888888888888888888888888888",
     "merge_commit_sha": null,
     "user_notes_count": 1,
-    "changes_count": "1",
     "should_remove_source_branch": true,
     "force_remove_source_branch": false,
     "web_url": "http://example.com/example/example/merge_requests/1",
@@ -351,6 +339,11 @@ Parameters:
 
 Shows information about a single merge request.
 
+**Note**: the `changes_count` value in the response is a string, not an
+integer. This is because when an MR has too many changes to display and store,
+it will be capped at 1,000. In that case, the API will return the string
+`"1000+"` for the changes count.
+
 ```
 GET /projects/:id/merge_requests/:merge_request_iid
 ```

doc/development/fe_guide/performance.md

@@ -15,7 +15,7 @@ Use the following rules when creating realtime solutions.
 Use that as your polling interval. This way it is [easy for system administrators to change the
 polling rate](../../administration/polling.md).
 A `Poll-Interval: -1` means you should disable polling, and this must be implemented.
-1. A response with HTTP status `4XX` or `5XX` should disable polling as well.
+1. A response with HTTP status different from 2XX should disable polling as well.
 1. Use a common library for polling.
 1. Poll on active tabs only. Please use [Visibility](https://github.com/ai/visibilityjs).
 1. Use regular polling intervals, do not use backoff polling, or jitter, as the interval will be
@@ -25,15 +25,15 @@ controlled by the server.
 
 ### Lazy Loading Images
 
-To improve the time to first render we are using lazy loading for images. This works by setting 
-the actual image source on the `data-src` attribute. After the HTML is rendered and JavaScript is loaded, 
+To improve the time to first render we are using lazy loading for images. This works by setting
+the actual image source on the `data-src` attribute. After the HTML is rendered and JavaScript is loaded,
 the value of `data-src` will be moved to `src` automatically if the image is in the current viewport.
 
 *  Prepare images in HTML for lazy loading by renaming the `src` attribute to `data-src` AND adding the class `lazy`
 *  If you are using the Rails `image_tag` helper, all images will be lazy-loaded by default unless `lazy: false` is provided.
 
 If you are asynchronously adding content which contains lazy images then you need to call the function
-`gl.lazyLoader.searchLazyImages()` which will search for lazy images and load them if needed. 
+`gl.lazyLoader.searchLazyImages()` which will search for lazy images and load them if needed.
 But in general it should be handled automatically through a `MutationObserver` in the lazy loading function.
 
 ### Animations
@@ -97,19 +97,19 @@ bundle and included on the page.
 
     ```javascript
     import initMyWidget from './my_widget';
-  
+
     document.addEventListener('DOMContentLoaded', () => {
       initMyWidget();
     });
     ```
 
-- **Supporting Module Placement:**  
+- **Supporting Module Placement:**
     - If a class or a module is _specific to a particular route_, try to locate
       it close to the entry point it will be used. For instance, if
       `my_widget.js` is only imported within `pages/widget/show/index.js`, you
       should place the module at `pages/widget/show/my_widget.js` and import it
       with a relative path (e.g. `import initMyWidget from './my_widget';`).
-      
+
     - If a class or module is _used by multiple routes_, place it within a
       shared directory at the closest common parent directory for the entry
       points that import it.  For example, if `my_widget.js` is imported within

doc/university/high-availability/aws/README.md

@@ -2,10 +2,8 @@
 comments: false
 ---
 
-DANGER: This guide exists for reference of how an AWS deployment could work.
-We are currently seeing very slow EFS access performance which causes GitLab to
-be 5-10x slower than using NFS or Local disk. We _do not_ recommend follow this
-guide at this time.
+> **Note**: We **do not** recommend using the AWS Elastic File System (EFS), as it can result
+in [significantly degraded performance](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/administration/high_availability/nfs.md#aws-elastic-file-system).
 
 # High Availability on AWS
 

doc/user/admin_area/monitoring/health_check.md

@@ -20,14 +20,24 @@ To access monitoring resources, the client IP needs to be included in a whitelis
 
 [Read how to add IPs to a whitelist for the monitoring endpoints][admin].
 
-## Using the endpoint
+## Using the endpoints
 
 With default whitelist settings, the probes can be accessed from localhost:
 
+- `http://localhost/-/health`
 - `http://localhost/-/readiness`
 - `http://localhost/-/liveness`
 
-which will then provide a report of system health in JSON format.
+
+The first endpoint, `/-/health/`, only checks whether the application server is running. It does
+-not verify the database or other services are running. A successful response will return
+a 200 status code with the following message:
+
+```
+GitLab OK
+```
+
+The readiness and liveness probes will provide a report of system health in JSON format.
 
 Readiness example output:
 
@@ -42,12 +52,6 @@ Readiness example output:
    "shared_state_check" : {
       "status" : "ok"
    },
-   "fs_shards_check" : {
-      "labels" : {
-         "shard" : "default"
-      },
-      "status" : "ok"
-   },
    "db_check" : {
       "status" : "ok"
    },
@@ -61,9 +65,6 @@ Liveness example output:
 
 ```
 {
-   "fs_shards_check" : {
-      "status" : "ok"
-   },
    "cache_check" : {
       "status" : "ok"
    },

lib/api/entities.rb

@@ -1264,7 +1264,13 @@ module API
       end
 
       class Artifacts < Grape::Entity
-        expose :name, :untracked, :paths, :when, :expire_in
+        expose :name
+        expose :untracked
+        expose :paths
+        expose :when
+        expose :expire_in
+        expose :artifact_type
+        expose :artifact_format
       end
 
       class Cache < Grape::Entity

lib/api/members.rb

@@ -75,7 +75,10 @@ module API
           member = source.members.find_by(user_id: params[:user_id])
           conflict!('Member already exists') if member
 
-          member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
+          user = User.find_by_id(params[:user_id])
+          not_found!('User') unless user
+
+          member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at])
 
           if !member
             not_allowed! # This currently can only be reached in EE

lib/api/runner.rb

@@ -109,7 +109,7 @@ module API
         if result.valid?
           if result.build
             Gitlab::Metrics.add_event(:build_found)
-            present result.build, with: Entities::JobRequest::Response
+            present Ci::BuildRunnerPresenter.new(result.build), with: Entities::JobRequest::Response
           else
             Gitlab::Metrics.add_event(:build_not_found)
             header 'X-GitLab-Last-Update', new_update
@@ -231,6 +231,10 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :artifact_type, type: String, desc: %q(The type of artifact),
+                                 default: 'archive', values: Ci::JobArtifact.file_types.keys
+        optional :artifact_format, type: String, desc: %q(The format of artifact),
+                                   default: 'zip', values: Ci::JobArtifact.file_formats.keys
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
@@ -254,29 +258,29 @@ module API
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
-        bad_request!("Already uploaded") if job.job_artifacts_archive
-
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-        job.build_job_artifacts_archive(
+        job.job_artifacts.build(
           project: job.project,
           file: artifacts,
-          file_type: :archive,
+          file_type: params['artifact_type'],
+          file_format: params['artifact_format'],
           file_sha256: artifacts.sha256,
           expire_in: expire_in)
 
         if metadata
-          job.build_job_artifacts_metadata(
+          job.job_artifacts.build(
             project: job.project,
             file: metadata,
             file_type: :metadata,
+            file_format: :gzip,
             file_sha256: metadata.sha256,
             expire_in: expire_in)
         end
 
         if job.update(artifacts_expire_in: expire_in)
-          present job, with: Entities::JobRequest::Response
+          present Ci::BuildRunnerPresenter.new(job), with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
         end

lib/gitlab/auth/activity.rb

+module Gitlab
+  module Auth
+    ##
+    # Metrics and logging for user authentication activity.
+    #
+    class Activity
+      extend Gitlab::Utils::StrongMemoize
+
+      COUNTERS = {
+        user_authenticated: 'Counter of successful authentication events',
+        user_unauthenticated: 'Counter of authentication failures',
+        user_not_found: 'Counter of failed log-ins when user is unknown',
+        user_password_invalid: 'Counter of failed log-ins with invalid password',
+        user_session_override: 'Counter of manual log-ins and sessions overrides',
+        user_session_destroyed: 'Counter of user sessions being destroyed',
+        user_two_factor_authenticated: 'Counter of two factor authentications',
+        user_sessionless_authentication: 'Counter of sessionless authentications',
+        user_blocked: 'Counter of sign in attempts when user is blocked'
+      }.freeze
+
+      def initialize(user, opts)
+        @user = user
+        @opts = opts
+      end
+
+      def user_authentication_failed!
+        self.class.user_unauthenticated_counter_increment!
+
+        case @opts[:message]
+        when :not_found_in_database
+          self.class.user_not_found_counter_increment!
+        when :invalid
+          self.class.user_password_invalid_counter_increment!
+        end
+
+        self.class.user_blocked_counter_increment! if @user&.blocked?
+      end
+
+      def user_authenticated!
+        self.class.user_authenticated_counter_increment!
+      end
+
+      def user_session_override!
+        self.class.user_session_override_counter_increment!
+
+        case @opts[:message]
+        when :two_factor_authenticated
+          self.class.user_two_factor_authenticated_counter_increment!
+        when :sessionless_sign_in
+          self.class.user_sessionless_authentication_counter_increment!
+        end
+      end
+
+      def user_session_destroyed!
+        self.class.user_session_destroyed_counter_increment!
+      end
+
+      def self.each_counter
+        COUNTERS.each_pair do |metric, description|
+          yield "#{metric}_counter", metric, description
+        end
+      end
+
+      each_counter do |counter, metric, description|
+        define_singleton_method(counter) do
+          strong_memoize(counter) do
+            Gitlab::Metrics.counter("gitlab_auth_#{metric}_total".to_sym, description)
+          end
+        end
+
+        define_singleton_method("#{counter}_increment!") do
+          public_send(counter).increment # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+    end
+  end
+end

lib/gitlab/auth/blocked_user_tracker.rb

@@ -2,37 +2,58 @@
 module Gitlab
   module Auth
     class BlockedUserTracker
+      include Gitlab::Utils::StrongMemoize
+
       ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters'
 
-      def self.log_if_user_blocked(env)
-        message = env.dig('warden.options', :message)
+      def initialize(env)
+        @env = env
+      end
 
-        # Devise calls User#active_for_authentication? on the User model and then
-        # throws an exception to Warden with User#inactive_message:
-        # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8
-        #
-        # Since Warden doesn't pass the user record to the failure handler, we
-        # need to do a database lookup with the username. We can limit the
-        # lookups to happen when the user was blocked by checking the inactive
-        # message passed along by Warden.
-        return unless message == User::BLOCKED_MESSAGE
+      def user_blocked?
+        user&.blocked?
+      end
 
-        # Check for either LDAP or regular GitLab account logins
-        login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') ||
-          env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
+      def user
+        return unless has_user_blocked_message?
 
-        return unless login.present?
+        strong_memoize(:user) do
+          # Check for either LDAP or regular GitLab account logins
+          login = @env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') ||
+            @env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
 
-        user = User.by_login(login)
+          User.by_login(login) if login.present?
+        end
+      rescue TypeError
+      end
 
-        return unless user&.blocked?
+      def log_blocked_user_activity!
+        return unless user_blocked?
 
-        Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}")
+        Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{@env['REMOTE_ADDR']}")
         SystemHooksService.new.execute_hooks_for(user, :failed_login)
-
         true
       rescue TypeError
       end
+
+      private
+
+      ##
+      # Devise calls User#active_for_authentication? on the User model and then
+      # throws an exception to Warden with User#inactive_message:
+      # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8
+      #
+      # Since Warden doesn't pass the user record to the failure handler, we
+      # need to do a database lookup with the username. We can limit the
+      # lookups to happen when the user was blocked by checking the inactive
+      # message passed along by Warden.
+      #
+      def has_user_blocked_message?
+        strong_memoize(:user_blocked_message) do
+          message = @env.dig('warden.options', :message)
+          message == User::BLOCKED_MESSAGE
+        end
+      end
     end
   end
 end

lib/gitlab/ci/config/entry/artifacts.rb

@@ -6,13 +6,16 @@ module Gitlab
         # Entry that represents a configuration of job artifacts.
         #
         class Artifacts < Node
+          include Configurable
           include Validatable
           include Attributable
 
-          ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze
+          ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze
 
           attributes ALLOWED_KEYS
 
+          entry :reports, Entry::Reports, description: 'Report-type artifacts.'
+
           validations do
             validates :config, type: Hash
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -21,6 +24,7 @@ module Gitlab
               validates :name, type: String
               validates :untracked, boolean: true
               validates :paths, array_of_strings: true
+              validates :reports, type: Hash
               validates :when,
                 inclusion: { in: %w[on_success on_failure always],
                              message: 'should be on_success, on_failure ' \
@@ -28,6 +32,13 @@ module Gitlab
               validates :expire_in, duration: true
             end
           end
+
+          helpers :reports
+
+          def value
+            @config[:reports] = reports_value if @config.key?(:reports)
+            @config
+          end
         end
       end
     end

lib/gitlab/ci/config/entry/commands.rb

@@ -9,18 +9,7 @@ module Gitlab
           include Validatable
 
           validations do
-            include LegacyValidationHelpers
-
-            validate do
-              unless string_or_array_of_strings?(config)
-                errors.add(:config,
-                           'should be a string or an array of strings')
-              end
-            end
-
-            def string_or_array_of_strings?(field)
-              validate_string(field) || validate_array_of_strings(field)
-            end
+            validates :config, array_of_strings_or_string: true
           end
 
           def value

lib/gitlab/ci/config/entry/reports.rb

+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of job artifacts.
+        #
+        class Reports < Node
+          include Validatable
+          include Attributable
+
+          ALLOWED_KEYS = %i[junit].freeze
+
+          attributes ALLOWED_KEYS
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :junit, array_of_strings_or_string: true
+            end
+          end
+
+          def value
+            @config.transform_values { |v| Array(v) }
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/validators.rb

@@ -130,6 +130,20 @@ module Gitlab
             end
           end
 
+          class ArrayOfStringsOrStringValidator < RegexpValidator
+            def validate_each(record, attribute, value)
+              unless validate_array_of_strings_or_string(value)
+                record.errors.add(attribute, 'should be an array of strings or a string')
+              end
+            end
+
+            private
+
+            def validate_array_of_strings_or_string(values)
+              validate_array_of_strings(values) || validate_string(values)
+            end
+          end
+
           class TypeValidator < ActiveModel::EachValidator
             def validate_each(record, attribute, value)
               type = options[:with]

lib/gitlab/ci/trace.rb

@@ -164,6 +164,8 @@ module Gitlab
 
       def create_build_trace!(job, path)
         File.open(path) do |stream|
+          # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
+          # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20307
           job.create_job_artifacts_trace!(
             project: job.project,
             file_type: :trace,

lib/gitlab/git_post_receive.rb

+# frozen_string_literal: true
+
 module Gitlab
   class GitPostReceive
     include Gitlab::Identifier
@@ -14,10 +16,11 @@ module Gitlab
     end
 
     def changes_refs
-      return enum_for(:changes_refs) unless block_given?
+      return changes unless block_given?
 
       changes.each do |change|
-        oldrev, newrev, ref = change.strip.split(' ')
+        change.strip!
+        oldrev, newrev, ref = change.split(' ')
 
         yield oldrev, newrev, ref
       end
@@ -26,13 +29,10 @@ module Gitlab
     private
 
     def deserialize_changes(changes)
-      changes = utf8_encode_changes(changes)
-      changes.lines
+      utf8_encode_changes(changes).each_line
     end
 
     def utf8_encode_changes(changes)
-      changes = changes.dup
-
       changes.force_encoding('UTF-8')
       return changes if changes.valid_encoding?
 

lib/gitlab/gpg.rb

@@ -71,8 +71,16 @@ module Gitlab
       if MUTEX.locked? && MUTEX.owned?
         optimistic_using_tmp_keychain(&block)
       else
-        MUTEX.synchronize do
-          optimistic_using_tmp_keychain(&block)
+        if Gitlab.rails5?
+          ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+            MUTEX.synchronize do
+              optimistic_using_tmp_keychain(&block)
+            end
+          end
+        else
+          MUTEX.synchronize do
+            optimistic_using_tmp_keychain(&block)
+          end
         end
       end
     end

lib/gitlab/middleware/basic_health_check.rb

+# frozen_string_literal: true
+
+# This middleware provides a health check that does not hit the database. Its purpose
+# is to notify the prober that the application server is handling requests, but a 200
+# response does not signify that the database or other services are ready.
+#
+# See https://thisdata.com/blog/making-a-rails-health-check-that-doesnt-hit-the-database/ for
+# more details.
+
+module Gitlab
+  module Middleware
+    class BasicHealthCheck
+      # This can't be frozen because Rails::Rack::Logger wraps the body
+      # rubocop:disable Style/MutableConstant
+      OK_RESPONSE = [200, { 'Content-Type' => 'text/plain' }, ["GitLab OK"]]
+      EMPTY_RESPONSE = [404, { 'Content-Type' => 'text/plain' }, [""]]
+      # rubocop:enable Style/MutableConstant
+      HEALTH_PATH = '/-/health'
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH
+
+        request = Rack::Request.new(env)
+
+        return OK_RESPONSE if client_ip_whitelisted?(request)
+
+        EMPTY_RESPONSE
+      end
+
+      def client_ip_whitelisted?(request)
+        ip_whitelist.any? { |e| e.include?(request.ip) }
+      end
+
+      def ip_whitelist
+        @ip_whitelist ||= Settings.monitoring.ip_whitelist.map(&IPAddr.method(:new))
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -652,6 +652,9 @@ msgstr ""
 msgid "Are you sure you want to delete this pipeline schedule?"
 msgstr ""
 
+msgid "Are you sure you want to lose unsaved changes?"
+msgstr ""
+
 msgid "Are you sure you want to remove %{group_name}?"
 msgstr ""
 

spec/controllers/application_controller_spec.rb

@@ -57,6 +57,10 @@ describe ApplicationController do
   end
 
   describe "#authenticate_user_from_personal_access_token!" do
+    before do
+      stub_authentication_activity_metrics(debug: false)
+    end
+
     controller(described_class) do
       def index
         render text: 'authenticated'
@@ -67,7 +71,13 @@ describe ApplicationController do
 
     context "when the 'personal_access_token' param is populated with the personal access token" do
       it "logs the user in" do
+        expect(authentication_metrics)
+          .to increment(:user_authenticated_counter)
+          .and increment(:user_session_override_counter)
+          .and increment(:user_sessionless_authentication_counter)
+
         get :index, private_token: personal_access_token.token
+
         expect(response).to have_gitlab_http_status(200)
         expect(response.body).to eq('authenticated')
       end
@@ -75,15 +85,25 @@ describe ApplicationController do
 
     context "when the 'PERSONAL_ACCESS_TOKEN' header is populated with the personal access token" do
       it "logs the user in" do
+        expect(authentication_metrics)
+          .to increment(:user_authenticated_counter)
+          .and increment(:user_session_override_counter)
+          .and increment(:user_sessionless_authentication_counter)
+
         @request.headers["PRIVATE-TOKEN"] = personal_access_token.token
         get :index
+
         expect(response).to have_gitlab_http_status(200)
         expect(response.body).to eq('authenticated')
       end
     end
 
     it "doesn't log the user in otherwise" do
+      expect(authentication_metrics)
+        .to increment(:user_unauthenticated_counter)
+
       get :index, private_token: "token"
+
       expect(response.status).not_to eq(200)
       expect(response.body).not_to eq('authenticated')
     end
@@ -174,6 +194,10 @@ describe ApplicationController do
   end
 
   describe '#authenticate_sessionless_user!' do
+    before do
+      stub_authentication_activity_metrics(debug: false)
+    end
+
     describe 'authenticating a user from a feed token' do
       controller(described_class) do
         def index
@@ -184,7 +208,13 @@ describe ApplicationController do
       context "when the 'feed_token' param is populated with the feed token" do
         context 'when the request format is atom' do
           it "logs the user in" do
+            expect(authentication_metrics)
+              .to increment(:user_authenticated_counter)
+              .and increment(:user_session_override_counter)
+              .and increment(:user_sessionless_authentication_counter)
+
             get :index, feed_token: user.feed_token, format: :atom
+
             expect(response).to have_gitlab_http_status 200
             expect(response.body).to eq 'authenticated'
           end
@@ -192,7 +222,13 @@ describe ApplicationController do
 
         context 'when the request format is ics' do
           it "logs the user in" do
+            expect(authentication_metrics)
+              .to increment(:user_authenticated_counter)
+              .and increment(:user_session_override_counter)
+              .and increment(:user_sessionless_authentication_counter)
+
             get :index, feed_token: user.feed_token, format: :ics
+
             expect(response).to have_gitlab_http_status 200
             expect(response.body).to eq 'authenticated'
           end
@@ -200,7 +236,11 @@ describe ApplicationController do
 
         context 'when the request format is neither atom nor ics' do
           it "doesn't log the user in" do
+            expect(authentication_metrics)
+              .to increment(:user_unauthenticated_counter)
+
             get :index, feed_token: user.feed_token
+
             expect(response.status).not_to have_gitlab_http_status 200
             expect(response.body).not_to eq 'authenticated'
           end
@@ -209,7 +249,11 @@ describe ApplicationController do
 
       context "when the 'feed_token' param is populated with an invalid feed token" do
         it "doesn't log the user" do
+          expect(authentication_metrics)
+            .to increment(:user_unauthenticated_counter)
+
           get :index, feed_token: 'token', format: :atom
+
           expect(response.status).not_to eq 200
           expect(response.body).not_to eq 'authenticated'
         end

spec/controllers/projects/wikis_controller_spec.rb

@@ -2,50 +2,131 @@ require 'spec_helper'
 
 describe Projects::WikisController do
   let(:project) { create(:project, :public, :repository) }
-  let(:user) { create(:user) }
-  let(:wiki) { ProjectWiki.new(project, user) }
+  let(:user) { project.owner }
+  let(:project_wiki) { ProjectWiki.new(project, user) }
+  let(:wiki) { project_wiki.wiki }
+  let(:wiki_title) { 'page-title-test' }
 
-  describe 'GET #show' do
-    let(:wiki_title) { 'page-title-test' }
+  before do
+    create_page(wiki_title, 'hello world')
+
+    sign_in(user)
+  end
+
+  after do
+    destroy_page(wiki_title)
+  end
 
+  describe 'GET #show' do
     render_views
 
-    before do
-      create_page(wiki_title, 'hello world')
-    end
+    subject { get :show, namespace_id: project.namespace, project_id: project, id: wiki_title }
 
-    it 'limits the retrieved pages for the sidebar' do
-      sign_in(user)
+    context 'when page content encoding is invalid' do
+      it 'limits the retrieved pages for the sidebar' do
+        expect(controller).to receive(:load_wiki).and_return(project_wiki)
 
-      expect(controller).to receive(:load_wiki).and_return(wiki)
+        # empty? call
+        expect(project_wiki).to receive(:pages).with(limit: 1).and_call_original
+        # Sidebar entries
+        expect(project_wiki).to receive(:pages).with(limit: 15).and_call_original
 
-      # empty? call
-      expect(wiki).to receive(:pages).with(limit: 1).and_call_original
-      # Sidebar entries
-      expect(wiki).to receive(:pages).with(limit: 15).and_call_original
+        subject
+
+        expect(response).to have_http_status(:ok)
+        expect(response.body).to include(wiki_title)
+      end
+    end
 
-      get :show, namespace_id: project.namespace, project_id: project, id: wiki_title
+    context 'when page content encoding is invalid' do
+      it 'sets flash error' do
+        allow(controller).to receive(:valid_encoding?).and_return(false)
 
-      expect(response).to have_http_status(:ok)
-      expect(response.body).to include(wiki_title)
+        subject
+
+        expect(response).to have_http_status(:ok)
+        expect(flash[:notice]).to eq 'The content of this page is not encoded in UTF-8. Edits can only be made via the Git repository.'
+      end
     end
   end
 
   describe 'POST #preview_markdown' do
     it 'renders json in a correct format' do
-      sign_in(user)
-
       post :preview_markdown, namespace_id: project.namespace, project_id: project, id: 'page/path', text: '*Markdown* text'
 
       expect(JSON.parse(response.body).keys).to match_array(%w(body references))
     end
   end
 
+  describe 'GET #edit' do
+    subject { get(:edit, namespace_id: project.namespace, project_id: project, id: wiki_title) }
+
+    context 'when page content encoding is invalid' do
+      it 'redirects to show' do
+        allow(controller).to receive(:valid_encoding?).and_return(false)
+
+        subject
+
+        expect(response).to redirect_to(project_wiki_path(project, project_wiki.pages.first))
+      end
+    end
+
+    context 'when page content encoding is valid' do
+      render_views
+
+      it 'shows the edit page' do
+        subject
+
+        expect(response).to have_http_status(:ok)
+        expect(response.body).to include('Edit Page')
+      end
+    end
+  end
+
+  describe 'PATCH #update' do
+    let(:new_title) { 'New title' }
+    let(:new_content) { 'New content' }
+    subject do
+      patch(:update,
+            namespace_id: project.namespace,
+            project_id: project,
+            id: wiki_title,
+            wiki: { title: new_title, content: new_content })
+    end
+
+    context 'when page content encoding is invalid' do
+      it 'redirects to show' do
+        allow(controller).to receive(:valid_encoding?).and_return(false)
+
+        subject
+        expect(response).to redirect_to(project_wiki_path(project, project_wiki.pages.first))
+      end
+    end
+
+    context 'when page content encoding is valid' do
+      render_views
+
+      it 'updates the page' do
+        subject
+
+        wiki_page = project_wiki.pages.first
+
+        expect(wiki_page.title).to eq new_title
+        expect(wiki_page.content).to eq new_content
+      end
+    end
+  end
+
   def create_page(name, content)
-    project.wiki.wiki.write_page(name, :markdown, content, commit_details(name))
+    wiki.write_page(name, :markdown, content, commit_details(name))
   end
 
   def commit_details(name)
     Gitlab::Git::Wiki::CommitDetails.new(user.id, user.username, user.name, user.email, "created page #{name}")
   end
+
+  def destroy_page(title, dir = '')
+    page = wiki.page(title: title, dir: dir)
+    project_wiki.delete_page(page, "test commit")
+  end
 end

spec/factories/ci/builds.rb

@@ -187,6 +187,13 @@ FactoryBot.define do
       end
     end
 
+    trait :test_reports do
+      after(:create) do |build|
+        create(:ci_job_artifact, :junit, job: build)
+        build.reload
+      end
+    end
+
     trait :expired do
       artifacts_expire_at 1.minute.ago
     end

spec/factories/ci/job_artifacts.rb

@@ -4,6 +4,7 @@ FactoryBot.define do
   factory :ci_job_artifact, class: Ci::JobArtifact do
     job factory: :ci_build
     file_type :archive
+    file_format :zip
 
     trait :remote_store do
       file_store JobArtifactUploader::Store::REMOTE
@@ -15,6 +16,7 @@ FactoryBot.define do
 
     trait :archive do
       file_type :archive
+      file_format :zip
 
       after(:build) do |artifact, _|
         artifact.file = fixture_file_upload(
@@ -24,6 +26,7 @@ FactoryBot.define do
 
     trait :metadata do
       file_type :metadata
+      file_format :gzip
 
       after(:build) do |artifact, _|
         artifact.file = fixture_file_upload(
@@ -33,6 +36,7 @@ FactoryBot.define do
 
     trait :trace do
       file_type :trace
+      file_format :raw
 
       after(:build) do |artifact, evaluator|
         artifact.file = fixture_file_upload(
@@ -40,6 +44,16 @@ FactoryBot.define do
       end
     end
 
+    trait :junit do
+      file_type :junit
+      file_format :gzip
+
+      after(:build) do |artifact, evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/junit.xml.gz'), 'application/x-gzip')
+      end
+    end
+
     trait :correct_checksum do
       after(:build) do |artifact, evaluator|
         artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest

spec/features/projects/wiki/user_views_wiki_page_spec.rb

@@ -137,6 +137,26 @@ describe 'User views a wiki page' do
       end
     end
 
+    context 'when page has invalid content encoding' do
+      let(:content) { 'whatever'.force_encoding('ISO-8859-1') }
+
+      before do
+        allow(Gitlab::EncodingHelper).to receive(:encode!).and_return(content)
+
+        visit(project_wiki_path(project, wiki_page))
+      end
+
+      it 'does not show "Edit" button' do
+        expect(page).not_to have_selector('a.btn', text: 'Edit')
+      end
+
+      it 'shows error' do
+        page.within(:css, '.flash-notice') do
+          expect(page).to have_content('The content of this page is not encoded in UTF-8. Edits can only be made via the Git repository.')
+        end
+      end
+    end
+
     it 'opens a default wiki page', :js do
       visit(project_path(project))