Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ee into ee-psimyn-issue-note-refac

.gitlab-ci.yml

@@ -604,7 +604,7 @@ codequality:
   script:
     - cp .rubocop.yml .rubocop.yml.bak
     - grep -v "rubocop-gitlab-security" .rubocop.yml.bak > .rubocop.yml
-    - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 analyze -f json > raw_codeclimate.json
+    - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate analyze -f json > raw_codeclimate.json
     - cat raw_codeclimate.json | docker run -i stedolan/jq -c 'map({check_name,fingerprint,location})' > codeclimate.json
     - mv .rubocop.yml.bak .rubocop.yml
   artifacts:

Gemfile

@@ -114,7 +114,7 @@ gem 'google-api-client', '~> 0.13.6'
 gem 'unf', '~> 0.1.4'
 
 # Seed data
-gem 'seed-fu', '~> 2.3.5'
+gem 'seed-fu', '~> 2.3.7'
 
 # Search
 gem 'elasticsearch-model', '~> 0.1.9'
@@ -295,7 +295,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~> 0.7.0.beta37'
+  gem 'prometheus-client-mmap', '~> 0.7.0.beta39'
   gem 'raindrops', '~> 0.18'
 end
 

Gemfile.lock

@@ -654,7 +654,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.7.0.beta37)
+    prometheus-client-mmap (0.7.0.beta39)
       mmap2 (~> 2.2, >= 2.2.9)
     pry (0.10.4)
       coderay (~> 1.1.0)
@@ -844,7 +844,7 @@ GEM
       rake (>= 0.9, < 13)
       sass (~> 3.4.20)
     securecompare (1.0.0)
-    seed-fu (2.3.6)
+    seed-fu (2.3.7)
       activerecord (>= 3.1)
       activesupport (>= 3.1)
     select2-rails (3.5.9.3)
@@ -1149,7 +1149,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.7.0.beta37)
+  prometheus-client-mmap (~> 0.7.0.beta39)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

app/assets/javascripts/issue_show/components/app.vue

@@ -16,6 +16,10 @@ export default {
       required: true,
       type: String,
     },
+    updateEndpoint: {
+      required: true,
+      type: String,
+    },
     canUpdate: {
       required: true,
       type: Boolean,
@@ -262,6 +266,8 @@ export default {
         :description-text="state.descriptionText"
         :updated-at="state.updatedAt"
         :task-status="state.taskStatus"
+        :issuable-type="issuableType"
+        :update-url="updateEndpoint"
       />
       <edited-component
         v-if="hasUpdated"

app/assets/javascripts/issue_show/components/description.vue

@@ -22,6 +22,16 @@
         required: false,
         default: '',
       },
+      issuableType: {
+        type: String,
+        required: false,
+        default: 'issue',
+      },
+      updateUrl: {
+        type: String,
+        required: false,
+        default: null,
+      },
     },
     data() {
       return {
@@ -48,7 +58,7 @@
         if (this.canUpdate) {
           // eslint-disable-next-line no-new
           new TaskList({
-            dataType: 'issue',
+            dataType: this.issuableType,
             fieldName: 'description',
             selector: '.detail-page-description',
           });
@@ -95,7 +105,9 @@
     <textarea
       class="hidden js-task-list-field"
       v-if="descriptionText"
-      v-model="descriptionText">
+      v-model="descriptionText"
+      :data-update-url="updateUrl"
+    >
     </textarea>
   </div>
 </template>

app/assets/javascripts/lib/utils/text_utility.js

@@ -56,9 +56,11 @@ export const slugify = str => str.trim().toLowerCase();
 export const truncate = (string, maxLength) => `${string.substr(0, (maxLength - 3))}...`;
 
 /**
- * Capitalizes first character.
+ * Capitalizes first character
  *
  * @param {String} text
- * @returns {String}
+ * @return {String}
  */
-export const capitalizeFirstCharacter = text => `${text[0].toUpperCase()}${text.slice(1)}`;
+export function capitalizeFirstCharacter(text) {
+  return `${text[0].toUpperCase()}${text.slice(1)}`;
+}

app/assets/javascripts/vue_shared/components/icon.vue

@@ -12,6 +12,9 @@
   />
 
 */
+  // only allow classes in images.scss e.g. s12
+  const validSizes = [8, 12, 16, 18, 24, 32, 48, 72];
+
   export default {
     props: {
       name: {
@@ -23,6 +26,9 @@
         type: Number,
         required: false,
         default: 16,
+        validator(value) {
+          return validSizes.includes(value);
+        },
       },
 
       cssClasses: {
@@ -42,6 +48,7 @@
     },
   };
 </script>
+
 <template>
   <svg
     :class="[iconSizeClass, cssClasses]">

app/assets/stylesheets/pages/issuable.scss

@@ -292,6 +292,8 @@
     .gutter-toggle {
       margin-top: 7px;
       border-left: 1px solid $border-gray-normal;
+      padding-left: 0;
+      text-align: center;
     }
 
     .title .gutter-toggle {

app/controllers/concerns/issuable_actions.rb

@@ -54,7 +54,7 @@ module IssuableActions
   end
 
   def destroy
-    issuable.destroy
+    Issuable::DestroyService.new(issuable.project, current_user).execute(issuable)
     TodoService.new.destroy_issuable(issuable, current_user)
 
     name = issuable.human_class_name

app/controllers/projects/commits_controller.rb

@@ -45,8 +45,7 @@ class Projects::CommitsController < Projects::ApplicationController
   private
 
   def set_commits
-    render_404 unless request.format == :atom || @repository.blob_at(@commit.id, @path) || @repository.tree(@commit.id, @path).entries.present?
-
+    render_404 unless @path.empty? || request.format == :atom || @repository.blob_at(@commit.id, @path) || @repository.tree(@commit.id, @path).entries.present?
     @limit, @offset = (params[:limit] || 40).to_i, (params[:offset] || 0).to_i
     search = params[:search]
 

app/finders/runner_jobs_finder.rb

+class RunnerJobsFinder
+  attr_reader :runner, :params
+
+  def initialize(runner, params = {})
+    @runner = runner
+    @params = params
+  end
+
+  def execute
+    items = @runner.builds
+    items = by_status(items)
+    items
+  end
+
+  private
+
+  def by_status(items)
+    return items unless HasStatus::AVAILABLE_STATUSES.include?(params[:status])
+
+    items.where(status: params[:status])
+  end
+end

app/helpers/application_settings_helper.rb

@@ -178,6 +178,9 @@ module ApplicationSettingsHelper
       :ed25519_key_restriction,
       :email_author_in_body,
       :enabled_git_access_protocol,
+      :gitaly_timeout_default,
+      :gitaly_timeout_medium,
+      :gitaly_timeout_fast,
       :gravatar_enabled,
       :hashed_storage_enabled,
       :help_page_hide_commercial_content,

app/helpers/issuables_helper.rb

@@ -213,6 +213,7 @@ module IssuablesHelper
   def issuable_initial_data(issuable)
     data = {
       endpoint: issuable_path(issuable),
+      updateEndpoint: "#{issuable_path(issuable)}.json",
       canUpdate: can?(current_user, :"update_#{issuable.to_ability_name}", issuable),
       canDestroy: can?(current_user, :"destroy_#{issuable.to_ability_name}", issuable),
       canAdmin: can?(current_user, :"admin_#{issuable.to_ability_name}", issuable),

app/models/application_setting.rb

@@ -185,6 +185,27 @@ class ApplicationSetting < ActiveRecord::Base
     end
   end
 
+  validates :gitaly_timeout_default,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
+  validates :gitaly_timeout_medium,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+  validates :gitaly_timeout_medium,
+            numericality: { less_than_or_equal_to: :gitaly_timeout_default },
+            if: :gitaly_timeout_default
+  validates :gitaly_timeout_medium,
+            numericality: { greater_than_or_equal_to: :gitaly_timeout_fast },
+            if: :gitaly_timeout_fast
+
+  validates :gitaly_timeout_fast,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+  validates :gitaly_timeout_fast,
+            numericality: { less_than_or_equal_to: :gitaly_timeout_default },
+            if: :gitaly_timeout_default
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end
@@ -325,7 +346,10 @@ class ApplicationSetting < ActiveRecord::Base
       slack_app_enabled: false,
       slack_app_id: nil,
       slack_app_secret: nil,
-      slack_app_verification_token: nil
+      slack_app_verification_token: nil,
+      gitaly_timeout_fast: 10,
+      gitaly_timeout_medium: 30,
+      gitaly_timeout_default: 55
     }
   end
 

app/models/ci/build.rb

@@ -108,6 +108,7 @@ module Ci
       end
 
       before_transition any => [:failed] do |build|
+        next unless build.project
         next if build.retries_max.zero?
 
         if build.retries_count < build.retries_max

app/models/ci/variable.rb

@@ -4,7 +4,6 @@ module Ci
     include HasVariable
     include Presentable
     prepend EE::Ci::Variable
-    include Presentable
 
     belongs_to :project
 

app/models/commit_status.rb

@@ -17,6 +17,7 @@ class CommitStatus < ActiveRecord::Base
   validates :name, presence: true, unless: :importing?
 
   alias_attribute :author, :user
+  alias_attribute :pipeline_id, :commit_id
 
   scope :failed_but_allowed, -> do
     where(allow_failure: true, status: [:failed, :canceled])
@@ -103,26 +104,29 @@ class CommitStatus < ActiveRecord::Base
     end
 
     after_transition do |commit_status, transition|
+      next unless commit_status.project
       next if transition.loopback?
 
       commit_status.run_after_commit do
-        if pipeline
+        if pipeline_id
           if complete? || manual?
-            PipelineProcessWorker.perform_async(pipeline.id)
+            PipelineProcessWorker.perform_async(pipeline_id)
           else
-            PipelineUpdateWorker.perform_async(pipeline.id)
+            PipelineUpdateWorker.perform_async(pipeline_id)
           end
         end
 
-        StageUpdateWorker.perform_async(commit_status.stage_id)
-        ExpireJobCacheWorker.perform_async(commit_status.id)
+        StageUpdateWorker.perform_async(stage_id)
+        ExpireJobCacheWorker.perform_async(id)
       end
     end
 
     after_transition any => :failed do |commit_status|
+      next unless commit_status.project
+
       commit_status.run_after_commit do
         MergeRequests::AddTodoWhenBuildFailsService
-          .new(pipeline.project, nil).execute(self)
+          .new(project, nil).execute(self)
       end
     end
   end

app/models/concerns/has_variable.rb

@@ -16,6 +16,10 @@ module HasVariable
        key: Gitlab::Application.secrets.db_key_base,
        algorithm: 'aes-256-cbc'
 
+    def key=(new_key)
+      super(new_key.to_s.strip)
+    end
+
     def to_runner_variable
       { key: key, value: value, public: false }
     end

app/models/concerns/issuable.rb

@@ -265,8 +265,10 @@ module Issuable
     participants(user).include?(user)
   end
 
-  def to_hook_data(user, old_labels: [], old_assignees: [], old_total_time_spent: nil)
+  def to_hook_data(user, old_associations: {})
     changes = previous_changes
+    old_labels = old_associations.fetch(:labels, [])
+    old_assignees = old_associations.fetch(:assignees, [])
 
     if old_labels != labels
       changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)]
@@ -280,9 +282,13 @@ module Issuable
       end
     end
 
-    if self.respond_to?(:total_time_spent) && old_total_time_spent != total_time_spent
+    if self.respond_to?(:total_time_spent)
+      old_total_time_spent = old_associations.fetch(:total_time_spent, nil)
+
+      if old_total_time_spent != total_time_spent
         changes[:total_time_spent] = [old_total_time_spent, total_time_spent]
       end
+    end
 
     Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes)
   end

app/models/geo/event_log.rb

@@ -30,6 +30,10 @@ module Geo
       class_name: 'Geo::LfsObjectDeletedEvent',
       foreign_key: :lfs_object_deleted_event_id
 
+    belongs_to :hashed_storage_attachments_event,
+      class_name: 'Geo::HashedStorageAttachmentsEvent',
+      foreign_key: :hashed_storage_attachments_event_id
+
     def self.latest_event
       order(id: :desc).first
     end
@@ -41,7 +45,8 @@ module Geo
         repository_renamed_event ||
         repositories_changed_event ||
         hashed_storage_migrated_event ||
-        lfs_object_deleted_event
+        lfs_object_deleted_event ||
+        hashed_storage_attachments_event
     end
 
     def project_id

app/models/geo/fdw/project.rb

+module Geo
+  module Fdw
+    class Project < ::Geo::BaseFdw
+      self.table_name = Gitlab::Geo.fdw_table('projects')
+    end
+  end
+end

app/models/geo/hashed_storage_attachments_event.rb

+module Geo
+  class HashedStorageAttachmentsEvent < ActiveRecord::Base
+    include Geo::Model
+
+    belongs_to :project
+
+    validates :project, :old_attachments_path, :new_attachments_path, presence: true
+  end
+end

app/models/geo_node.rb

@@ -150,27 +150,6 @@ class GeoNode < ActiveRecord::Base
     end
   end
 
-  # These are projects that meet the project restriction but haven't yet been
-  # synced (i.e., do not yet have a project registry entry).
-  #
-  # This query requires data from two different databases, and unavoidably
-  # plucks a list of project IDs from one into the other. This will not scale
-  # well with the number of synchronized projects - the query will increase
-  # linearly in size - so this should be replaced with postgres_fdw ASAP.
-  def unsynced_projects
-    registry_project_ids = project_registries.pluck(:project_id)
-    return projects if registry_project_ids.empty?
-
-    joined_relation = projects.joins(<<~SQL)
-      LEFT OUTER JOIN
-      (VALUES #{registry_project_ids.map { |id| "(#{id}, 't')" }.join(',')})
-      project_registry(project_id, registry_present)
-      ON projects.id = project_registry.project_id
-    SQL
-
-    joined_relation.where(project_registry: { registry_present: [nil, false] })
-  end
-
   def uploads
     if restricted_project_ids
       uploads_table   = Upload.arel_table

app/models/issue.rb

@@ -64,7 +64,6 @@ class Issue < ActiveRecord::Base
   scope :public_only, -> { where(confidential: false) }
 
   after_save :expire_etag_cache
-  after_commit :update_project_counter_caches, on: :destroy
 
   attr_spammable :title, spam_title: true
   attr_spammable :description, spam_description: true

app/models/merge_request.rb

@@ -56,7 +56,6 @@ class MergeRequest < ActiveRecord::Base
 
   after_create :ensure_merge_request_diff, unless: :importing?
   after_update :reload_diff_if_branch_changed
-  after_commit :update_project_counter_caches, on: :destroy
 
   # When this attribute is true some MR validation is ignored
   # It allows us to close or modify broken merge requests

app/models/project.rb

@@ -277,8 +277,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
-  scope :with_hashed_storage, -> { where('storage_version >= 1') }
-  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+  scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) }
 
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }

app/models/storage/hashed_project.rb

@@ -4,7 +4,6 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
-    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project

app/policies/group_policy.rb

@@ -45,6 +45,8 @@ class GroupPolicy < BasePolicy
   rule { admin }             .enable :read_group
   rule { has_projects }      .enable :read_group
 
+  rule { has_access }.enable :read_namespace
+
   rule { developer }.enable :admin_milestones
   rule { reporter }.enable :admin_label
 

app/policies/namespace_policy.rb

@@ -8,6 +8,7 @@ class NamespacePolicy < BasePolicy
   rule { owner | admin }.policy do
     enable :create_projects
     enable :admin_namespace
+    enable :read_namespace
   end
 
   rule { personal_project & ~can_create_personal_project }.prevent :create_projects

app/services/geo/hashed_storage_attachments_event_store.rb

+module Geo
+  class HashedStorageAttachmentsEventStore < EventStore
+    self.event_type = :hashed_storage_attachments_event
+
+    private
+
+    def build_event
+      Geo::HashedStorageAttachmentsEvent.new(
+        project: project,
+        old_attachments_path: old_attachments_path,
+        new_attachments_path: new_attachments_path
+      )
+    end
+
+    def old_attachments_path
+      params.fetch(:old_attachments_path)
+    end
+
+    def new_attachments_path
+      params.fetch(:new_attachments_path)
+    end
+  end
+end

app/services/geo/hashed_storage_attachments_migration_service.rb

+module Geo
+  AttachmentMigrationError = Class.new(StandardError)
+
+  class HashedStorageAttachmentsMigrationService
+    include ::Gitlab::Geo::LogHelpers
+
+    attr_reader :project_id, :old_attachments_path, :new_attachments_path
+
+    def initialize(project_id, old_attachments_path:, new_attachments_path:)
+      @project_id = project_id
+      @old_attachments_path = old_attachments_path
+      @new_attachments_path = new_attachments_path
+    end
+
+    def async_execute
+      Geo::HashedStorageAttachmentsMigrationWorker.perform_async(
+        project_id,
+        old_attachments_path,
+        new_attachments_path
+      )
+    end
+
+    def execute
+      origin = File.join(CarrierWave.root, FileUploader.base_dir, old_attachments_path)
+      target = File.join(CarrierWave.root, FileUploader.base_dir, new_attachments_path)
+      move_folder!(origin, target)
+    end
+
+    private
+
+    def project
+      @project ||= Project.find(project_id)
+    end
+
+    def move_folder!(old_path, new_path)
+      unless File.directory?(old_path)
+        log_info("Skipped attachments migration to Hashed Storage, source path doesn't exist or is not a directory", project_id: project.id, source: old_path, target: new_path)
+        return
+      end
+
+      if File.exist?(new_path)
+        log_error("Cannot migrate attachments to Hashed Storage, target path already exist", project_id: project.id, source: old_path, target: new_path)
+        raise AttachmentMigrationError, "Target path '#{new_path}' already exist"
+      end
+
+      # Create hashed storage base path folder
+      FileUtils.mkdir_p(File.dirname(new_path))
+
+      FileUtils.mv(old_path, new_path)
+      log_info("Migrated project attachments to Hashed Storage", project_id: project.id, source: old_path, target: new_path)
+
+      true
+    end
+  end
+end

app/services/geo/hashed_storage_migration_service.rb

 module Geo
   class HashedStorageMigrationService
+    include ::Gitlab::Geo::LogHelpers
+
     attr_reader :project_id, :old_disk_path, :new_disk_path, :old_storage_version
 
     def initialize(project_id, old_disk_path:, new_disk_path:, old_storage_version:)
@@ -22,9 +24,12 @@ module Geo
       project.expire_caches_before_rename(old_disk_path)
 
       if migrating_from_legacy_storage? && !move_repository
+        log_error("Repository could not be migrated to Hashed Storage", project_id: project.id, source: old_disk_path, target: new_disk_path)
         raise RepositoryCannotBeRenamed, "Repository #{old_disk_path} could not be renamed to #{new_disk_path}"
       end
 
+      log_info("Repository migrated to Hashed Storage", project_id: project.id, source: old_disk_path, target: new_disk_path)
+
       true
     end
 

app/services/geo/wiki_sync_service.rb

@@ -37,7 +37,7 @@ module Geo
     rescue Gitlab::Git::Repository::NoRepository => e
       log_error('Invalid wiki', e)
       registry.update(force_to_redownload_wiki: true,
-                      repository_retry_count: retry_count + 1)
+                      wiki_retry_count: retry_count + 1)
     ensure
       clean_up_temporary_repository if redownload
     end

app/services/issuable/destroy_service.rb

+module Issuable
+  class DestroyService < IssuableBaseService
+    def execute(issuable)
+      if issuable.destroy
+        issuable.update_project_counter_caches
+      end
+    end
+  end
+end

app/services/issuable_base_service.rb

@@ -165,16 +165,13 @@ class IssuableBaseService < BaseService
     # To be overridden by subclasses
   end
 
-  def update(issuable) # rubocop:disable Metrics/AbcSize
+  def update(issuable)
     change_state(issuable)
     change_subscription(issuable)
     change_todo(issuable)
     toggle_award(issuable)
     filter_params(issuable)
-    old_labels = issuable.labels.to_a
-    old_mentioned_users = issuable.mentioned_users.to_a
-    old_assignees = issuable.assignees.to_a
-    old_total_time_spent = issuable.total_time_spent if issuable.respond_to?(:total_time_spent)
+    old_associations = associations_before_update(issuable)
 
     label_ids = process_label_ids(params, existing_label_ids: issuable.label_ids)
     params[:label_ids] = label_ids if labels_changing?(issuable.label_ids, label_ids)
@@ -195,18 +192,13 @@ class IssuableBaseService < BaseService
       if issuable.with_transaction_returning_status { issuable.save }
         # We do not touch as it will affect a update on updated_at field
         ActiveRecord::Base.no_touching do
-          Issuable::CommonSystemNotesService.new(project, current_user).execute(issuable, old_labels)
+          Issuable::CommonSystemNotesService.new(project, current_user).execute(issuable, old_associations[:labels])
         end
 
-        handle_changes(
-          issuable,
-          old_labels: old_labels,
-          old_mentioned_users: old_mentioned_users,
-          old_assignees: old_assignees
-        )
+        handle_changes(issuable, old_associations: old_associations)
 
         new_assignees = issuable.assignees.to_a
-        affected_assignees = (old_assignees + new_assignees) - (old_assignees & new_assignees)
+        affected_assignees = (old_associations[:assignees] + new_assignees) - (old_associations[:assignees] & new_assignees)
 
         invalidate_cache_counts(issuable, users: affected_assignees.compact)
         after_update(issuable)
@@ -214,9 +206,8 @@ class IssuableBaseService < BaseService
         execute_hooks(
           issuable,
           'update',
-          old_labels: old_labels,
-          old_assignees: old_assignees,
-          old_total_time_spent: old_total_time_spent)
+          old_associations: old_associations
+        )
 
         issuable.update_project_counter_caches if update_project_counters
       end
@@ -269,6 +260,18 @@ class IssuableBaseService < BaseService
     end
   end
 
+  def associations_before_update(issuable)
+    associations =
+      {
+        labels: issuable.labels.to_a,
+        mentioned_users: issuable.mentioned_users.to_a,
+        assignees: issuable.assignees.to_a
+      }
+    associations[:total_time_spent] = issuable.total_time_spent if issuable.respond_to?(:total_time_spent)
+
+    associations
+  end
+
   def has_changes?(issuable, old_labels: [], old_assignees: [])
     valid_attrs = [:title, :description, :assignee_id, :milestone_id, :target_branch]
 

app/services/issues/base_service.rb

 module Issues
   class BaseService < ::IssuableBaseService
-    def hook_data(issue, action, old_labels: [], old_assignees: [], old_total_time_spent: nil)
-      hook_data = issue.to_hook_data(current_user, old_labels: old_labels, old_assignees: old_assignees, old_total_time_spent: old_total_time_spent)
+    def hook_data(issue, action, old_associations: {})
+      hook_data = issue.to_hook_data(current_user, old_associations: old_associations)
       hook_data[:object_attributes][:action] = action
 
       hook_data
@@ -22,8 +22,8 @@ module Issues
         issue, issue.project, current_user, old_assignees)
     end
 
-    def execute_hooks(issue, action = 'open', old_labels: [], old_assignees: [], old_total_time_spent: nil)
-      issue_data  = hook_data(issue, action, old_labels: old_labels, old_assignees: old_assignees, old_total_time_spent: old_total_time_spent)
+    def execute_hooks(issue, action = 'open', old_associations: {})
+      issue_data  = hook_data(issue, action, old_associations: old_associations)
       hooks_scope = issue.confidential? ? :confidential_issue_hooks : :issue_hooks
       issue.project.execute_hooks(issue_data, hooks_scope)
       issue.project.execute_services(issue_data, hooks_scope)

app/services/issues/update_service.rb

@@ -14,9 +14,10 @@ module Issues
     end
 
     def handle_changes(issue, options)
-      old_labels = options[:old_labels] || []
-      old_mentioned_users = options[:old_mentioned_users] || []
-      old_assignees = options[:old_assignees] || []
+      old_associations = options.fetch(:old_associations, {})
+      old_labels = old_associations.fetch(:labels, [])
+      old_mentioned_users = old_associations.fetch(:mentioned_users, [])
+      old_assignees = old_associations.fetch(:assignees, [])
 
       if has_changes?(issue, old_labels: old_labels, old_assignees: old_assignees)
         todo_service.mark_pending_todos_as_done(issue, current_user)

app/services/merge_requests/base_service.rb

@@ -6,8 +6,8 @@ module MergeRequests
       SystemNoteService.change_status(merge_request, merge_request.target_project, current_user, state, nil)
     end
 
-    def hook_data(merge_request, action, old_rev: nil, old_labels: [], old_assignees: [], old_total_time_spent: nil)
-      hook_data = merge_request.to_hook_data(current_user, old_labels: old_labels, old_assignees: old_assignees, old_total_time_spent: old_total_time_spent)
+    def hook_data(merge_request, action, old_rev: nil, old_associations: {})
+      hook_data = merge_request.to_hook_data(current_user, old_associations: old_associations)
       hook_data[:object_attributes][:action] = action
       if old_rev && !Gitlab::Git.blank_ref?(old_rev)
         hook_data[:object_attributes][:oldrev] = old_rev
@@ -16,9 +16,9 @@ module MergeRequests
       hook_data
     end
 
-    def execute_hooks(merge_request, action = 'open', old_rev: nil, old_labels: [], old_assignees: [], old_total_time_spent: nil)
+    def execute_hooks(merge_request, action = 'open', old_rev: nil, old_associations: {})
       if merge_request.project
-        merge_data = hook_data(merge_request, action, old_rev: old_rev, old_labels: old_labels, old_assignees: old_assignees, old_total_time_spent: old_total_time_spent)
+        merge_data = hook_data(merge_request, action, old_rev: old_rev, old_associations: old_associations)
         merge_request.project.execute_hooks(merge_data, :merge_request_hooks)
         merge_request.project.execute_services(merge_data, :merge_request_hooks)
       end

app/services/merge_requests/update_service.rb

@@ -33,8 +33,9 @@ module MergeRequests
     end
 
     def handle_changes(merge_request, options)
-      old_labels = options[:old_labels] || []
-      old_mentioned_users = options[:old_mentioned_users] || []
+      old_associations = options.fetch(:old_associations, {})
+      old_labels = old_associations.fetch(:labels, [])
+      old_mentioned_users = old_associations.fetch(:mentioned_users, [])
 
       if has_changes?(merge_request, old_labels: old_labels)
         todo_service.mark_pending_todos_as_done(merge_request, current_user)

app/services/projects/hashed_storage/migrate_attachments_service.rb

+module Projects
+  module HashedStorage
+    AttachmentMigrationError = Class.new(StandardError)
+
+    class MigrateAttachmentsService < BaseService
+      attr_reader :logger, :old_path, :new_path
+
+      prepend ::EE::Projects::HashedStorage::MigrateAttachmentsService
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_path = project.full_path
+        @new_path = project.disk_path
+
+        origin = FileUploader.dynamic_path_segment(project)
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        target = FileUploader.dynamic_path_segment(project)
+
+        result = move_folder!(origin, target)
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_folder!(old_path, new_path)
+        unless File.directory?(old_path)
+          logger.info("Skipped attachments migration from '#{old_path}' to '#{new_path}', source path doesn't exist or is not a directory (PROJECT_ID=#{project.id})")
+          return
+        end
+
+        if File.exist?(new_path)
+          logger.error("Cannot migrate attachments from '#{old_path}' to '#{new_path}', target path already exist (PROJECT_ID=#{project.id})")
+          raise AttachmentMigrationError, "Target path '#{new_path}' already exist"
+        end
+
+        # Create hashed storage base path folder
+        FileUtils.mkdir_p(File.dirname(new_path))
+
+        FileUtils.mv(old_path, new_path)
+        logger.info("Migrated project attachments from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})")
+
+        true
+      end
+    end
+  end
+end

app/services/projects/hashed_storage/migrate_repository_service.rb

+module Projects
+  module HashedStorage
+    class MigrateRepositoryService < BaseService
+      include Gitlab::ShellAdapter
+
+      prepend ::EE::Projects::HashedStorage::MigrateRepositoryService
+
+      attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version, :logger
+
+      def initialize(project, logger = nil)
+        @project = project
+        @logger = logger || Rails.logger
+      end
+
+      def execute
+        @old_disk_path = project.disk_path
+        has_wiki = project.wiki.repository_exists?
+
+        @old_storage_version = project.storage_version
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
+        project.ensure_storage_path_exists
+
+        @new_disk_path = project.disk_path
+
+        result = move_repository(@old_disk_path, @new_disk_path)
+
+        if has_wiki
+          @old_wiki_disk_path = "#{@old_disk_path}.wiki"
+          result &&= move_repository("#{@old_wiki_disk_path}", "#{@new_disk_path}.wiki")
+        end
+
+        unless result
+          rollback_folder_move
+          project.storage_version = nil
+        end
+
+        project.repository_read_only = false
+        project.save!
+
+        if result && block_given?
+          yield
+        end
+
+        result
+      end
+
+      private
+
+      def move_repository(from_name, to_name)
+        from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
+        to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+
+        # If we don't find the repository on either original or target we should log that as it could be an issue if the
+        # project was not originally empty.
+        if !from_exists && !to_exists
+          logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
+          return false
+        elsif !from_exists
+          # Repository have been moved already.
+          return true
+        end
+
+        gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+      end
+
+      def rollback_folder_move
+        move_repository(@new_disk_path, @old_disk_path)
+        move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
+      end
+    end
+  end
+end

app/services/projects/hashed_storage_migration_service.rb

 module Projects
   class HashedStorageMigrationService < BaseService
-    include Gitlab::ShellAdapter
-
-    prepend ::EE::Projects::HashedStorageMigrationService
-
-    attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version
+    attr_reader :logger
 
     def initialize(project, logger = nil)
       @project = project
-      @logger ||= Rails.logger
+      @logger = logger || Rails.logger
     end
 
     def execute
-      return if project.hashed_storage?(:repository)
-
-      @old_disk_path = project.disk_path
-      has_wiki = project.wiki.repository_exists?
-
-      @old_storage_version = project.storage_version
-      project.storage_version = Storage::HashedProject::STORAGE_VERSION
-      project.ensure_storage_path_exists
-
-      @new_disk_path = project.disk_path
-
-      result = move_repository(@old_disk_path, @new_disk_path)
-
-      if has_wiki
-        @old_wiki_disk_path = "#{@old_disk_path}.wiki"
-        result &&= move_repository(@old_wiki_disk_path, "#{@new_disk_path}.wiki")
+      # Migrate repository from Legacy to Hashed Storage
+      unless project.hashed_storage?(:repository)
+        return unless HashedStorage::MigrateRepositoryService.new(project, logger).execute
       end
 
-      unless result
-        rollback_folder_move
-        return
-      end
-
-      project.repository_read_only = false
-      project.save!
-
-      block_given? ? yield : result
+      # Migrate attachments from Legacy to Hashed Storage
+      unless project.hashed_storage?(:attachments)
+        HashedStorage::MigrateAttachmentsService.new(project, logger).execute
       end
-
-    private
-
-    def move_repository(from_name, to_name)
-      from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
-      to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
-
-      # If we don't find the repository on either original or target we should log that as it could be an issue if the
-      # project was not originally empty.
-      if !from_exists && !to_exists
-        logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
-        return false
-      elsif !from_exists
-        # Repository have been moved already.
-        return true
-      end
-
-      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
-    end
-
-    def rollback_folder_move
-      move_repository(@new_disk_path, @old_disk_path)
-      move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
-    end
-
-    def logger
-      @logger
     end
   end
 end

app/uploaders/file_uploader.rb

@@ -31,12 +31,19 @@ class FileUploader < GitlabUploader
   # Returns a String without a trailing slash
   def self.dynamic_path_segment(project)
     if project.hashed_storage?(:attachments)
-      File.join(CarrierWave.root, base_dir, project.disk_path)
+      dynamic_path_builder(project.disk_path)
     else
-      File.join(CarrierWave.root, base_dir, project.full_path)
+      dynamic_path_builder(project.full_path)
     end
   end
 
+  # Auxiliary method to build dynamic path segment when not using a project model
+  #
+  # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic
+  def self.dynamic_path_builder(path)
+    File.join(CarrierWave.root, base_dir, path)
+  end
+
   attr_accessor :model
   attr_reader :secret
 

app/views/admin/application_settings/_form.html.haml

@@ -771,6 +771,30 @@
         .help-block
           Number of Git pushes after which 'git gc' is run.
 
+  %fieldset
+    %legend Gitaly Timeouts
+    .form-group
+      = f.label :gitaly_timeout_default, 'Default Timeout Period', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :gitaly_timeout_default, class: 'form-control'
+        .help-block
+          Timeout for Gitaly calls from the GitLab application (in seconds). This timeout is not enforced
+          for git fetch/push operations or Sidekiq jobs.
+    .form-group
+      = f.label :gitaly_timeout_fast, 'Fast Timeout Period', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :gitaly_timeout_fast, class: 'form-control'
+        .help-block
+          Fast operation timeout (in seconds). Some Gitaly operations are expected to be fast.
+          If they exceed this threshold, there may be a problem with a storage shard and 'failing fast'
+          can help maintain the stability of the GitLab instance.
+    .form-group
+      = f.label :gitaly_timeout_medium, 'Medium Timeout Period', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :gitaly_timeout_medium, class: 'form-control'
+        .help-block
+          Medium operation timeout (in seconds). This should be a value between the Fast and the Default timeout.
+
   %fieldset
     %legend Web terminal
     .form-group

app/workers/geo/file_download_dispatch_worker.rb

@@ -13,7 +13,7 @@ module Geo
     end
 
     def finder
-      @finder ||= RegistryFinder.new(current_node: current_node)
+      @finder ||= FileRegistryFinder.new(current_node: current_node)
     end
 
     # Pools for new resources to be transferred

app/workers/geo/hashed_storage_attachments_migration_worker.rb

+module Geo
+  class HashedStorageAttachmentsMigrationWorker
+    include Sidekiq::Worker
+    include GeoQueue
+
+    def perform(project_id, old_attachments_path, new_attachments_path)
+      Geo::HashedStorageAttachmentsMigrationService.new(
+        project_id,
+        old_attachments_path: old_attachments_path,
+        new_attachments_path: new_attachments_path
+      ).execute
+    end
+  end
+end

app/workers/geo/repository_sync_worker.rb

@@ -12,6 +12,10 @@ module Geo
       { id: project_id, job_id: job_id } if job_id
     end
 
+    def finder
+      @finder ||= ProjectRegistryFinder.new(current_node: current_node)
+    end
+
     def load_pending_resources
       resources = find_project_ids_not_synced(batch_size: db_retrieve_batch_size)
       remaining_capacity = db_retrieve_batch_size - resources.size
@@ -24,19 +28,15 @@ module Geo
     end
 
     def find_project_ids_not_synced(batch_size:)
-      healthy_shards_restriction(current_node.unsynced_projects)
+      healthy_shards_restriction(finder.find_unsynced_projects(batch_size: batch_size))
         .reorder(last_repository_updated_at: :desc)
-        .limit(batch_size)
         .pluck(:id)
     end
 
     def find_project_ids_updated_recently(batch_size:)
-      current_node.project_registries
-                  .dirty
-                  .retry_due
-                  .order(Gitlab::Database.nulls_first_order(:last_repository_synced_at, :desc))
-                  .limit(batch_size)
-                  .pluck(:project_id)
+      healthy_shards_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
+        .order(Gitlab::Database.nulls_first_order(:last_repository_updated_at, :desc))
+        .pluck(:id)
     end
 
     def healthy_shards_restriction(relation)

app/workers/project_migrate_hashed_storage_worker.rb

@@ -2,10 +2,34 @@ class ProjectMigrateHashedStorageWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  LEASE_TIMEOUT = 30.seconds.to_i
+
   def perform(project_id)
     project = Project.find_by(id: project_id)
     return if project.nil? || project.pending_delete?
 
+    uuid = lease_for(project_id).try_obtain
+    if uuid
       ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    else
+      false
+    end
+  rescue => ex
+    cancel_lease_for(project_id, uuid) if uuid
+    raise ex
+  end
+
+  def lease_for(project_id)
+    Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
+  end
+
+  private
+
+  def lease_key(project_id)
+    "project_migrate_hashed_storage_worker:#{project_id}"
+  end
+
+  def cancel_lease_for(project_id, uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
   end
 end

app/workers/stuck_ci_jobs_worker.rb

@@ -45,9 +45,17 @@ class StuckCiJobsWorker
   end
 
   def search(status, timeout)
-    builds = Ci::Build.where(status: status).where('ci_builds.updated_at < ?', timeout.ago)
-    builds.joins(:project).merge(Project.without_deleted).includes(:tags, :runner, project: :namespace).find_each(batch_size: 50).each do |build|
-      yield(build)
+    loop do
+      jobs = Ci::Build.where(status: status)
+        .where('ci_builds.updated_at < ?', timeout.ago)
+        .includes(:tags, :runner, project: :namespace)
+        .limit(100)
+        .to_a
+      break if jobs.empty?
+
+      jobs.each do |job|
+        yield(job)
+      end
     end
   end
 

changelogs/unreleased-ee/da-repository-sync-should-not-sync-repos-unhealthy-shards.yml

+---
+title: Geo - Does not sync repositories on unhealthy shards in non-backfill conditions
+merge_request:
+author:
+type: fixed

changelogs/unreleased-ee/ee-40561-environment-scope-value-is-not-trimmed.yml

+---
+title: Strip leading & trailing whitespaces in CI/CD secret variable's environment
+  scope
+merge_request: 3563
+author:
+type: fixed

changelogs/unreleased-ee/ee-hashed-storage-attachments-migration-path.yml

+---
+title: 'Geo: replicate Attachments migration to Hashed Storage in secondary node'
+merge_request: 3544
+author:
+type: added

changelogs/unreleased-ee/fix-epic-tasklist.yml

+---
+title: Fix tasklist for epics
+merge_request:
+author:
+type: fixed

changelogs/unreleased-ee/fix-geo-wiki-error.yml

+---
+title: Fix Geo wiki sync error not increasing retry count
+merge_request:
+author:
+type: fixed

changelogs/unreleased/39601-create-issuable-destroy-service.yml

+---
+title: Create issuable destroy service
+merge_request: 15604
+author: George Andrinopoulos
+type: other

changelogs/unreleased/40561-environment-scope-value-is-not-trimmed.yml

+---
+title: Strip leading & trailing whitespaces in CI/CD secret variable keys
+merge_request: 15615
+author:
+type: fixed

changelogs/unreleased/40568-bump-seed-fu-to-2-3-7.yml

+---
+title: Upgrade seed-fu to 2.3.7
+merge_request: 15607
+author: Takuya Noguchi
+type: other

changelogs/unreleased/an-gitaly-timeouts.yml

+---
+title: Add timeouts for Gitaly calls
+merge_request: 15047
+author:
+type: performance

changelogs/unreleased/hashed-storage-attachments-migration-path.yml

+---
+title: Hashed Storage migration script now supports migrating project attachments
+merge_request: 15352
+author:
+type: added

changelogs/unreleased/optimise-stuck-ci-jobs-worker.yml

+---
+title: Optimise StuckCiJobsWorker using cheap SQL query outside, and expensive inside
+merge_request:
+author:
+type: performance

changelogs/unreleased/tm-feature-list-runners-jobs-api.yml

+---
+title: New API endpoint - list jobs for a specified runner
+merge_request: 15432
+author:
+type: added

changelogs/unreleased/tm-feature-namespace-by-id-api.yml

+---
+title: Add new API endpoint - get a namespace by ID
+merge_request: 15442
+author:
+type: added

db/migrate/20171101130535_add_gitaly_timeout_properties_to_application_settings.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddGitalyTimeoutPropertiesToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings,
+                            :gitaly_timeout_default,
+                            :integer,
+                            default: 55
+    add_column_with_default :application_settings,
+                            :gitaly_timeout_medium,
+                            :integer,
+                            default: 30
+    add_column_with_default :application_settings,
+                            :gitaly_timeout_fast,
+                            :integer,
+                            default: 10
+  end
+
+  def down
+    remove_column :application_settings, :gitaly_timeout_default
+    remove_column :application_settings, :gitaly_timeout_medium
+    remove_column :application_settings, :gitaly_timeout_fast
+  end
+end

db/migrate/20171124070437_add_attachments_migration_to_geo_migration_events.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAttachmentsMigrationToGeoMigrationEvents < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :geo_hashed_storage_attachments_events, id: :bigserial do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+      t.text :old_attachments_path, null: false
+      t.text :new_attachments_path, null: false
+    end
+
+    add_column :geo_event_log, :hashed_storage_attachments_event_id, :integer, limit: 8
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171121144800) do
+ActiveRecord::Schema.define(version: 20171124070437) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -174,6 +174,9 @@ ActiveRecord::Schema.define(version: 20171121144800) do
     t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false
     t.boolean "password_authentication_enabled_for_web"
     t.boolean "password_authentication_enabled_for_git", default: true
+    t.integer "gitaly_timeout_default", default: 55, null: false
+    t.integer "gitaly_timeout_medium", default: 30, null: false
+    t.integer "gitaly_timeout_fast", default: 10, null: false
   end
 
   create_table "approvals", force: :cascade do |t|
@@ -882,6 +885,7 @@ ActiveRecord::Schema.define(version: 20171121144800) do
     t.integer "repository_created_event_id", limit: 8
     t.integer "hashed_storage_migrated_event_id", limit: 8
     t.integer "lfs_object_deleted_event_id", limit: 8
+    t.integer "hashed_storage_attachments_event_id", limit: 8
   end
 
   add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", using: :btree
@@ -890,6 +894,14 @@ ActiveRecord::Schema.define(version: 20171121144800) do
   add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", using: :btree
   add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
 
+  create_table "geo_hashed_storage_attachments_events", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.text "old_attachments_path", null: false
+    t.text "new_attachments_path", null: false
+  end
+
+  add_index "geo_hashed_storage_attachments_events", ["project_id"], name: "index_geo_hashed_storage_attachments_events_on_project_id", using: :btree
+
   create_table "geo_hashed_storage_migrated_events", id: :bigserial, force: :cascade do |t|
     t.integer "project_id", null: false
     t.text "repository_storage_name", null: false
@@ -2450,6 +2462,7 @@ ActiveRecord::Schema.define(version: 20171121144800) do
   add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
+  add_foreign_key "geo_hashed_storage_attachments_events", "projects", on_delete: :cascade
   add_foreign_key "geo_hashed_storage_migrated_events", "projects", on_delete: :cascade
   add_foreign_key "geo_node_namespace_links", "geo_nodes", on_delete: :cascade
   add_foreign_key "geo_node_namespace_links", "namespaces", on_delete: :cascade

doc/administration/high_availability/database.md

@@ -70,6 +70,16 @@ Database nodes run two services besides PostgreSQL
 
 Alongside pgbouncer, there is a consul agent that watches the status of the PostgreSQL service. If that status changes, consul runs a script which updates the configuration and reloads pgbouncer
 
+#### Connection flow
+
+Each service in the package comes with a set of [default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#ports). You may need to make specific firewall rules for the connections listed below:
+
+- Application servers connect to [PgBouncer default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#pgbouncer)
+- PgBouncer connects to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql)
+- Repmgr connects to the database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql)
+- Postgres secondaries connect to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql)
+- Consul servers and agents connect to each others [Consul default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#consul)
+
 ### Required information
 
 Before proceeding with configuration, you will need to collect all the necessary

doc/administration/raketasks/storage.md

 # Repository Storage Rake Tasks
 
 This is a collection of rake tasks you can use to help you list and migrate 
-existing projects from Legacy storage to the new Hashed storage type.
+existing projects and attachments associated with it from Legacy storage to 
+the new Hashed storage type.
 
 You can read more about the storage types [here][storage-types].
 
+## Migrate existing projects to Hashed storage
+
+Before migrating your existing projects, you should 
+[enable hashed storage][storage-migration] for the new projects as well.
+
+This task will schedule all your existing projects and attachments associated with it to be migrated to the 
+**Hashed** storage type:
+
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:migrate_to_hashed
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:migrate_to_hashed
+
+```
+
+You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
+There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage**
+
+After it reaches zero, you can confirm every project has been migrated by running the commands bellow. 
+If you find it necessary, you can run this migration script again to schedule missing projects.
+
+Any error or warning will be logged in the sidekiq's log file.
+
+You only need the `gitlab:storage:migrate_to_hashed` rake task to migrate your repositories, but we have additional
+commands below that helps you inspect projects and attachments in both legacy and hashed storage.
+
 ## List projects on Legacy storage
 
 To have a simple summary of projects using **Legacy** storage:
@@ -73,35 +106,73 @@ rake gitlab:storage:list_hashed_projects
 
 ```
 
-## Migrate existing projects to Hashed storage
+## List attachments on Legacy storage
 
-Before migrating your existing projects, you should 
-[enable hashed storage][storage-migration] for the new projects as well.
+To have a simple summary of project attachments using **Legacy** storage:
 
-This task will schedule all your existing projects to be migrated to the 
-**Hashed** storage type:
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:legacy_attachments
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:legacy_attachments
+
+```
+
+------
+
+To list project attachments using **Legacy** storage:
 
 **Omnibus Installation**
 
 ```bash
-gitlab-rake gitlab:storage:migrate_to_hashed
+gitlab-rake gitlab:storage:list_legacy_attachments
 ```
 
 **Source Installation**
 
 ```bash
-rake gitlab:storage:migrate_to_hashed
+rake gitlab:storage:list_legacy_attachments
 
 ```
 
-You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
-There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage**
+## List attachments on Hashed storage
 
-After it reaches zero, you can confirm every project has been migrated by running the commands above.
-If you find it necessary, you can run this migration script again to schedule missing projects.
+To have a simple summary of project attachments using **Hashed** storage:
+
+**Omnibus Installation**
+
+```bash
+gitlab-rake gitlab:storage:hashed_attachments
+```
 
-Any error or warning will be logged in the sidekiq log file.
+**Source Installation**
+
+```bash
+rake gitlab:storage:hashed_attachments
+
+```
+
+------
+
+To list project attachments using **Hashed** storage:
+
+**Omnibus Installation**
 
+```bash
+gitlab-rake gitlab:storage:list_hashed_attachments
+```
+
+**Source Installation**
+
+```bash
+rake gitlab:storage:list_hashed_attachments
+
+```
 
 [storage-types]: ../repository_storage_types.md
 [storage-migration]: ../repository_storage_types.md#how-to-migrate-to-hashed-storage

doc/api/namespaces.md

@@ -90,3 +90,55 @@ Example response:
   }
 ]
 ```
+
+## Get namespace by ID
+
+Get a namespace by ID.
+
+```
+GET /namespaces/:id
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | ID or path of the namespace |
+
+Example request:
+
+```bash
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/namespaces/2
+```
+
+Example response:
+
+```json
+{
+  "id": 2,
+  "name": "group1",
+  "path": "group1",
+  "kind": "group",
+  "full_path": "group1",
+  "parent_id": "null",
+  "members_count_with_descendants": 2
+}
+```
+
+Example request:
+
+```bash
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/namespaces/group1
+```
+
+Example response:
+
+```json
+{
+  "id": 2,
+  "name": "group1",
+  "path": "group1",
+  "kind": "group",
+  "full_path": "group1",
+  "parent_id": "null",
+  "members_count_with_descendants": 2
+}
+```

doc/api/runners.md

@@ -215,6 +215,91 @@ DELETE /runners/:id
 curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/runners/6"
 ```
 
+## List runner's jobs
+
+List jobs that are being processed or were processed by specified Runner.
+
+```
+GET /runners/:id/jobs
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `id`      | integer | yes      | The ID of a runner  |
+| `status`  | string  | no       | Status of the job; one of: `running`, `success`, `failed`, `canceled` |
+
+```
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/runners/1/jobs?status=running"
+```
+
+Example response:
+
+```json
+[
+    {
+        "id": 2,
+        "status": "running",
+        "stage": "test",
+        "name": "test",
+        "ref": "master",
+        "tag": false,
+        "coverage": null,
+        "created_at": "2017-11-16T08:50:29.000Z",
+        "started_at": "2017-11-16T08:51:29.000Z",
+        "finished_at": "2017-11-16T08:53:29.000Z",
+        "duration": 120,
+        "user": {
+            "id": 1,
+            "name": "John Doe2",
+            "username": "user2",
+            "state": "active",
+            "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon",
+            "web_url": "http://localhost/user2",
+            "created_at": "2017-11-16T18:38:46.000Z",
+            "bio": null,
+            "location": null,
+            "skype": "",
+            "linkedin": "",
+            "twitter": "",
+            "website_url": "",
+            "organization": null
+        },
+        "commit": {
+            "id": "97de212e80737a608d939f648d959671fb0a0142",
+            "short_id": "97de212e",
+            "title": "Update configuration\r",
+            "created_at": "2017-11-16T08:50:28.000Z",
+            "parent_ids": [
+                "1b12f15a11fc6e62177bef08f47bc7b5ce50b141",
+                "498214de67004b1da3d820901307bed2a68a8ef6"
+            ],
+            "message": "See merge request !123",
+            "author_name": "John Doe2",
+            "author_email": "user2@example.org",
+            "authored_date": "2017-11-16T08:50:27.000Z",
+            "committer_name": "John Doe2",
+            "committer_email": "user2@example.org",
+            "committed_date": "2017-11-16T08:50:27.000Z"
+        },
+        "pipeline": {
+            "id": 2,
+            "sha": "97de212e80737a608d939f648d959671fb0a0142",
+            "ref": "master",
+            "status": "running"
+        },
+        "project": {
+            "id": 1,
+            "description": null,
+            "name": "project1",
+            "name_with_namespace": "John Doe2 / project1",
+            "path": "project1",
+            "path_with_namespace": "namespace1/project1",
+            "created_at": "2017-11-16T18:38:46.620Z"
+        }
+    }
+]
+```
+
 ## List project's runners
 
 List all runners (specific and shared) available in the project. Shared runners

doc/user/project/clusters/index.md

@@ -82,7 +82,7 @@ added directly to your configured cluster. Those applications are needed for
 | Application | GitLab version | Description |
 | ----------- | :------------: | ----------- |
 | [Helm Tiller](https://docs.helm.sh/) | 10.2+ | Helm is a package manager for Kubernetes and is required to install all the other applications. It will be automatically installed as a dependency when you try to install a different app. It is installed in its own pod inside the cluster which can run the `helm` CLI in a safe environment. |
-| [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../../topics/autodevops/index.md) or deploy your own web apps. |
+| [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | 10.2+ | Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications and is useful if you want to use [Auto DevOps](../../../topics/autodevops/index.md) or deploy your own web apps. |
 
 ## Enabling or disabling the Cluster integration
 

ee/app/assets/javascripts/epics/epic_show/components/epic_show_app.vue

@@ -12,6 +12,10 @@
         type: String,
         required: true,
       },
+      updateEndpoint: {
+        type: String,
+        required: true,
+      },
       canUpdate: {
         required: true,
         type: Boolean,
@@ -111,7 +115,9 @@
           :can-update="canUpdate"
           :can-destroy="canDestroy"
           :endpoint="endpoint"
+          :update-endpoint="updateEndpoint"
           :issuable-ref="issuableRef"
+          issuable-type="epic"
           :initial-title-html="initialTitleHtml"
           :initial-title-text="initialTitleText"
           :initial-description-html="initialDescriptionHtml"

app/finders/geo/registry_finder.rb → ee/app/finders/geo/file_registry_finder.rb

 module Geo
-  class RegistryFinder
-    attr_reader :current_node
-
-    def initialize(current_node: nil)
-      @current_node = current_node
-    end
-
+  class FileRegistryFinder < RegistryFinder
     def find_failed_objects(batch_size:)
       Geo::FileRegistry
         .failed
@@ -28,7 +22,7 @@ module Geo
       # Selective project replication adds a wrinkle to FDW queries, so
       # we fallback to the legacy version for now.
       relation =
-        if Gitlab::Geo.fdw? && !selective_sync
+        if fdw?
           fdw_find_nonreplicated_lfs_objects
         else
           legacy_find_nonreplicated_lfs_objects(except_registry_ids: except_registry_ids)
@@ -54,7 +48,7 @@ module Geo
       # Selective project replication adds a wrinkle to FDW queries, so
       # we fallback to the legacy version for now.
       relation =
-        if Gitlab::Geo.fdw? && !selective_sync
+        if fdw?
           fdw_find_nonreplicated_uploads
         else
           legacy_find_nonreplicated_uploads(except_registry_ids: except_registry_ids)
@@ -68,10 +62,6 @@ module Geo
 
     protected
 
-    def selective_sync
-      current_node.restricted_project_ids
-    end
-
     #
     # FDW accessors
     #

ee/app/finders/geo/project_registry_finder.rb

+module Geo
+  class ProjectRegistryFinder < RegistryFinder
+    def find_unsynced_projects(batch_size:)
+      relation =
+        if fdw?
+          fdw_find_unsynced_projects
+        else
+          legacy_find_unsynced_projects
+        end
+
+      relation.limit(batch_size)
+    end
+
+    def find_projects_updated_recently(batch_size:)
+      relation =
+        if fdw?
+          fdw_find_projects_updated_recently
+        else
+          legacy_find_projects_updated_recently
+        end
+
+      relation.limit(batch_size)
+    end
+
+    protected
+
+    def fdw_table
+      Geo::Fdw::Project.table_name
+    end
+
+    #
+    # FDW accessors
+    #
+
+    # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
+    def fdw_find_unsynced_projects
+      Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
+        .where('project_registry.project_id IS NULL')
+    end
+
+    # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
+    def fdw_find_projects_updated_recently
+      Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
+        .merge(Geo::ProjectRegistry.dirty)
+        .merge(Geo::ProjectRegistry.retry_due)
+    end
+
+    #
+    # Legacy accessors (non FDW)
+    #
+
+    # @return [ActiveRecord::Relation<Project>] list of unsynced projects
+    def legacy_find_unsynced_projects
+      registry_project_ids = current_node.project_registries.pluck(:project_id)
+      return current_node.projects if registry_project_ids.empty?
+
+      joined_relation = current_node.projects.joins(<<~SQL)
+        LEFT OUTER JOIN
+        (VALUES #{registry_project_ids.map { |id| "(#{id}, 't')" }.join(',')})
+        project_registry(project_id, registry_present)
+        ON projects.id = project_registry.project_id
+      SQL
+
+      joined_relation.where(project_registry: { registry_present: [nil, false] })
+    end
+
+    # @return [ActiveRecord::Relation<Project>] list of projects updated recently
+    def legacy_find_projects_updated_recently
+      registry_project_ids = current_node.project_registries.dirty.retry_due.pluck(:project_id)
+      return Project.none if registry_project_ids.empty?
+
+      joined_relation = current_node.projects.joins(<<~SQL)
+        INNER JOIN
+        (VALUES #{registry_project_ids.map { |id| "(#{id})" }.join(',')})
+        project_registry(project_id)
+        ON projects.id = project_registry.project_id
+      SQL
+
+      joined_relation
+    end
+  end
+end

ee/app/finders/geo/registry_finder.rb

+module Geo
+  class RegistryFinder
+    attr_reader :current_node
+
+    def initialize(current_node: nil)
+      @current_node = current_node
+    end
+
+    protected
+
+    def fdw?
+      # Selective project replication adds a wrinkle to FDW
+      # queries, so we fallback to the legacy version for now.
+      Gitlab::Geo.fdw? && !selective_sync
+    end
+
+    def selective_sync
+      current_node.restricted_project_ids
+    end
+  end
+end

ee/app/models/ee/ci/variable.rb

@@ -11,6 +11,10 @@ module EE
                     message: ::Gitlab::Regex.environment_scope_regex_message }
         )
       end
+
+      def environment_scope=(new_environment_scope)
+        super(new_environment_scope.to_s.strip)
+      end
     end
   end
 end

ee/app/services/ee/projects/hashed_storage/migrate_attachments_service.rb

+module EE
+  module Projects
+    module HashedStorage
+      module MigrateAttachmentsService
+        def execute
+          raise NotImplementedError.new unless defined?(super)
+
+          super do
+            ::Geo::HashedStorageAttachmentsEventStore.new(
+              project,
+              old_attachments_path: old_path,
+              new_attachments_path: new_path
+            ).create
+          end
+        end
+      end
+    end
+  end
+end

ee/app/services/ee/projects/hashed_storage/migrate_repository_service.rb

+module EE
+  module Projects
+    module HashedStorage
+      module MigrateRepositoryService
+        def execute
+          raise NotImplementedError.new unless defined?(super)
+
+          super do
+            ::Geo::HashedStorageMigratedEventStore.new(
+              project,
+              old_storage_version: old_storage_version,
+              old_disk_path: old_disk_path,
+              old_wiki_disk_path: old_wiki_disk_path
+            ).create
+          end
+        end
+      end
+    end
+  end
+end

ee/app/services/ee/projects/hashed_storage_migration_service.rb

-module EE
-  module Projects
-    module HashedStorageMigrationService
-      def execute
-        raise NotImplementedError.new unless defined?(super)
-
-        super do
-          ::Geo::HashedStorageMigratedEventStore.new(
-            project,
-            old_storage_version: old_storage_version,
-            old_disk_path: old_disk_path,
-            old_wiki_disk_path: old_wiki_disk_path
-          ).create
-        end
-      end
-    end
-  end
-end

ee/app/services/projects/update_repository_storage_service.rb

@@ -35,7 +35,7 @@ module Projects
                                                    repository.relative_path,
                                                    repository.gl_repository)
 
-      new_repository.fetch_as_mirror_without_shell(repository.path)
+      new_repository.fetch_repository_as_mirror(repository)
     end
 
     def mark_old_paths_for_archive

lib/api/entities.rb

@@ -90,16 +90,21 @@ module API
       expose :group_access, as: :group_access_level
     end
 
-    class BasicProjectDetails < Grape::Entity
-      expose :id, :description, :default_branch, :tag_list
-      expose :ssh_url_to_repo, :http_url_to_repo, :web_url
+    class ProjectIdentity < Grape::Entity
+      expose :id, :description
       expose :name, :name_with_namespace
       expose :path, :path_with_namespace
+      expose :created_at
+    end
+
+    class BasicProjectDetails < ProjectIdentity
+      expose :default_branch, :tag_list
+      expose :ssh_url_to_repo, :http_url_to_repo, :web_url
       expose :avatar_url do |project, options|
         project.avatar_url(only_path: false)
       end
       expose :star_count, :forks_count
-      expose :created_at, :last_activity_at
+      expose :last_activity_at
     end
 
     class Project < BasicProjectDetails
@@ -938,17 +943,24 @@ module API
       expose :id, :sha, :ref, :status
     end
 
-    class Job < Grape::Entity
+    class JobBasic < Grape::Entity
       expose :id, :status, :stage, :name, :ref, :tag, :coverage
       expose :created_at, :started_at, :finished_at
       expose :duration
       expose :user, with: User
-      expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? }
       expose :commit, with: Commit
-      expose :runner, with: Runner
       expose :pipeline, with: PipelineBasic
     end
 
+    class Job < JobBasic
+      expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? }
+      expose :runner, with: Runner
+    end
+
+    class JobBasicWithProject < JobBasic
+      expose :project, with: ProjectIdentity
+    end
+
     class Trigger < Grape::Entity
       expose :id
       expose :token, :description

lib/api/helpers.rb

@@ -52,6 +52,10 @@ module API
       initial_current_user != current_user
     end
 
+    def user_namespace
+      @user_namespace ||= find_namespace!(params[:id])
+    end
+
     def user_group
       @group ||= find_group!(params[:id])
     end
@@ -109,14 +113,6 @@ module API
       end
     end
 
-    def find_namespace(id)
-      if id =~ /^\d+$/
-        Namespace.find_by(id: id)
-      else
-        Namespace.find_by_full_path(id)
-      end
-    end
-
     def find_group!(id)
       # CI job token authentication:
       # currently we do not allow any group access for CI job token
@@ -131,6 +127,24 @@ module API
       end
     end
 
+    def find_namespace(id)
+      if id.to_s =~ /^\d+$/
+        Namespace.find_by(id: id)
+      else
+        Namespace.find_by_full_path(id)
+      end
+    end
+
+    def find_namespace!(id)
+      namespace = find_namespace(id)
+
+      if can?(current_user, :read_namespace, namespace)
+        namespace
+      else
+        not_found!('Namespace')
+      end
+    end
+
     def find_project_label(id)
       label = available_labels.find_by_id(id) || available_labels.find_by_title(id)
       label || not_found!('Label')

lib/api/issues.rb

@@ -261,7 +261,9 @@ module API
 
         authorize!(:destroy_issue, issue)
 
-        destroy_conditionally!(issue)
+        destroy_conditionally!(issue) do |issue|
+          Issuable::DestroyService.new(user_project, current_user).execute(issue)
+        end
       end
 
       desc 'List merge requests closing issue'  do

lib/api/merge_requests.rb

@@ -179,7 +179,9 @@ module API
 
         authorize!(:destroy_merge_request, merge_request)
 
-        destroy_conditionally!(merge_request)
+        destroy_conditionally!(merge_request) do |merge_request|
+          Issuable::DestroyService.new(user_project, current_user).execute(merge_request)
+        end
       end
 
       params do

lib/api/namespaces.rb

@@ -40,6 +40,16 @@ module API
           render_validation_error!(namespace)
         end
       end
+
+      desc 'Get a namespace by ID' do
+        success Entities::Namespace
+      end
+      params do
+        requires :id, type: String, desc: "Namespace's ID or path"
+      end
+      get ':id' do
+        present user_namespace, with: Entities::Namespace, current_user: current_user
+      end
     end
   end
 end

lib/api/runners.rb

@@ -84,6 +84,23 @@ module API
 
         destroy_conditionally!(runner)
       end
+
+      desc 'List jobs running on a runner' do
+        success Entities::JobBasicWithProject
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the runner'
+        optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES
+        use :pagination
+      end
+      get  ':id/jobs' do
+        runner = get_runner(params[:id])
+        authenticate_list_runners_jobs!(runner)
+
+        jobs = RunnerJobsFinder.new(runner, params).execute
+
+        present paginate(jobs), with: Entities::JobBasicWithProject
+      end
     end
 
     params do
@@ -192,6 +209,12 @@ module API
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
+      def authenticate_list_runners_jobs!(runner)
+        return if current_user.admin?
+
+        forbidden!("No access granted") unless user_can_access_runner?(runner)
+      end
+
       def user_can_access_runner?(runner)
         current_user.ci_authorized_runners.exists?(runner.id)
       end

lib/api/settings.rb

@@ -123,6 +123,9 @@ module API
       end
       optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
       optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
+      optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
+      optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.'
+      optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.'
 
       ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
         optional :"#{type}_key_restriction",

lib/gitlab/geo/log_cursor/daemon.rb

@@ -183,6 +183,23 @@ module Gitlab
             job_id: job_id)
         end
 
+        def handle_hashed_storage_attachments_event(event, created_at)
+          job_id = ::Geo::HashedStorageAttachmentsMigrationService.new(
+            event.project_id,
+            old_attachments_path: event.old_attachments_path,
+            new_attachments_path: event.new_attachments_path
+          ).async_execute
+
+          logger.event_info(
+            created_at,
+            message: 'Migrating attachments to hashed storage',
+            project_id: event.project_id,
+            old_attachments_path: event.old_attachments_path,
+            new_attachments_path: event.new_attachments_path,
+            job_id: job_id
+          )
+        end
+
         def handle_lfs_object_deleted_event(event, created_at)
           file_path = File.join(LfsObjectUploader.local_store_path, event.file_path)
 

lib/gitlab/git/repository.rb

@@ -18,6 +18,7 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
+      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
 
       NoRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
@@ -1150,12 +1151,25 @@ module Gitlab
         @has_visible_content = has_local_branches?
       end
 
-      # Like all public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_remote_without_shell(remote = 'origin')
-        run_git(['fetch', remote]).last.zero?
+      def fetch_repository_as_mirror(repository)
+        remote_name = "tmp-#{SecureRandom.hex}"
+
+        # Notice that this feature flag is not for `fetch_repository_as_mirror`
+        # as a whole but for the fetching mechanism (file path or gitaly-ssh).
+        url, env = gitaly_migrate(:fetch_internal) do |is_enabled|
+          if is_enabled
+            repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository)
+            [GITALY_INTERNAL_URL, repository.fetch_env]
+          else
+            [repository.path, nil]
+          end
+        end
+
+        add_remote(remote_name, url)
+        set_remote_as_mirror(remote_name)
+        fetch_remote(remote_name, env: env)
+      ensure
+        remove_remote(remote_name)
       end
 
       def blob_at(sha, path)
@@ -1661,7 +1675,7 @@ module Gitlab
       end
 
       def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref})
+        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
 
         run_git(args, env: source_repository.fetch_env)
       end
@@ -1681,6 +1695,10 @@ module Gitlab
       rescue Rugged::ReferenceError
         raise ArgumentError, 'Invalid merge source'
       end
+
+      def fetch_remote(remote_name = 'origin', env: nil)
+        run_git(['fetch', remote_name], env: env).last.zero?
+      end
     end
   end
 end

lib/gitlab/git/repository_mirroring.rb

@@ -31,19 +31,6 @@ module Gitlab
         end
       end
 
-      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_as_mirror_without_shell(url)
-        remote_name = "tmp-#{SecureRandom.hex}"
-        add_remote(remote_name, url)
-        set_remote_as_mirror(remote_name)
-        fetch_remote_without_shell(remote_name)
-      ensure
-        remove_remote(remote_name) if remote_name
-      end
-
       def remote_tags(remote)
         # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
         # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]

lib/gitlab/gitaly_client.rb

@@ -93,11 +93,11 @@ module Gitlab
     #   kwargs.merge(deadline: Time.now + 10)
     # end
     #
-    def self.call(storage, service, rpc, request, remote_storage: nil)
+    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       enforce_gitaly_request_limits(:call)
 
-      kwargs = request_kwargs(storage, remote_storage: remote_storage)
+      kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
       kwargs = yield(kwargs) if block_given?
 
       stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
@@ -105,7 +105,7 @@ module Gitlab
       self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
     end
 
-    def self.request_kwargs(storage, remote_storage: nil)
+    def self.request_kwargs(storage, timeout, remote_storage: nil)
       encoded_token = Base64.strict_encode64(token(storage).to_s)
       metadata = {
         'authorization' => "Bearer #{encoded_token}",
@@ -117,7 +117,22 @@ module Gitlab
       metadata['call_site'] = feature.to_s if feature
       metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
 
-      { metadata: metadata }
+      result = { metadata: metadata }
+
+      # nil timeout indicates that we should use the default
+      timeout = default_timeout if timeout.nil?
+
+      return result unless timeout > 0
+
+      # Do not use `Time.now` for deadline calculation, since it
+      # will be affected by Timecop in some tests, but grpc's c-core
+      # uses system time instead of timecop's time, so tests will fail
+      # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will
+      # circumvent timecop
+      deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout
+      result[:deadline] = deadline
+
+      result
     end
 
     def self.token(storage)
@@ -290,6 +305,26 @@ module Gitlab
       Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } )
     end
 
+    # The default timeout on all Gitaly calls
+    def self.default_timeout
+      return 0 if Sidekiq.server?
+
+      timeout(:gitaly_timeout_default)
+    end
+
+    def self.fast_timeout
+      timeout(:gitaly_timeout_fast)
+    end
+
+    def self.medium_timeout
+      timeout(:gitaly_timeout_medium)
+    end
+
+    def self.timeout(timeout_name)
+      Gitlab::CurrentSettings.current_application_settings[timeout_name]
+    end
+    private_class_method :timeout
+
     # Count a stack. Used for n+1 detection
     def self.count_stack
       return unless RequestStore.active?

lib/gitlab/gitaly_client/commit_service.rb

@@ -16,7 +16,7 @@ module Gitlab
           revision: GitalyClient.encode(revision)
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout)
         response.flat_map do |msg|
           msg.paths.map { |d| EncodingHelper.encode!(d.dup) }
         end
@@ -29,7 +29,7 @@ module Gitlab
           child_id: child_id
         )
 
-        GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value
+        GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value
       end
 
       def diff(from, to, options = {})
@@ -77,7 +77,7 @@ module Gitlab
           limit: limit.to_i
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout)
 
         entry = nil
         data = ''
@@ -102,7 +102,7 @@ module Gitlab
           path: path.present? ? GitalyClient.encode(path) : '.'
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout)
 
         response.flat_map do |message|
           message.entries.map do |gitaly_tree_entry|
@@ -129,7 +129,7 @@ module Gitlab
         request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
         request.path = options[:path] if options[:path].present?
 
-        GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count
+        GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count
       end
 
       def last_commit_for_path(revision, path)
@@ -139,7 +139,7 @@ module Gitlab
           path: GitalyClient.encode(path.to_s)
         )
 
-        gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit
+        gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit
         return unless gitaly_commit
 
         Gitlab::Git::Commit.new(@repository, gitaly_commit)
@@ -152,7 +152,7 @@ module Gitlab
           to: to
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
       end
 
@@ -165,7 +165,7 @@ module Gitlab
         )
         request.order = opts[:order].upcase if opts[:order].present?
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
       end
 
@@ -179,7 +179,7 @@ module Gitlab
           offset: offset.to_i
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
       end
 
@@ -197,7 +197,7 @@ module Gitlab
           path: GitalyClient.encode(path)
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout)
         response.reduce("") { |memo, msg| memo << msg.data }
       end
 
@@ -207,7 +207,7 @@ module Gitlab
           revision: GitalyClient.encode(revision)
         )
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout)
 
         response.commit
       end
@@ -217,7 +217,7 @@ module Gitlab
           repository: @gitaly_repo,
           revision: GitalyClient.encode(revision)
         )
-        response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout)
 
         response.sum(&:data)
       end
@@ -227,7 +227,7 @@ module Gitlab
           repository: @gitaly_repo,
           revision: GitalyClient.encode(revision)
         )
-        GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request)
+        GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout)
       end
 
       def find_commits(options)
@@ -245,7 +245,7 @@ module Gitlab
 
         request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present?
 
-        response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request)
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout)
 
         consume_commits_response(response)
       end
@@ -259,7 +259,7 @@ module Gitlab
         request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h)
 
         request = Gitaly::CommitDiffRequest.new(request_params)
-        response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request)
+        response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout)
         GitalyClient::DiffStitcher.new(response)
       end
 

lib/gitlab/gitaly_client/ref_service.rb

@@ -46,7 +46,8 @@ module Gitlab
           commit_id: commit_id,
           prefix: ref_prefix
         )
-        encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup)
+        response = GitalyClient.call(@storage, :ref_service, :find_ref_name, request, timeout: GitalyClient.medium_timeout)
+        encode!(response.name.dup)
       end
 
       def count_tag_names

lib/gitlab/gitaly_client/repository_service.rb

@@ -10,7 +10,9 @@ module Gitlab
       def exists?
         request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo)
 
-        GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists
+        response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout)
+
+        response.exists
       end
 
       def garbage_collect(create_bitmap)
@@ -30,7 +32,8 @@ module Gitlab
 
       def repository_size
         request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo)
-        GitalyClient.call(@storage, :repository_service, :repository_size, request).size
+        response = GitalyClient.call(@storage, :repository_service, :repository_size, request)
+        response.size
       end
 
       def apply_gitattributes(revision)
@@ -61,7 +64,7 @@ module Gitlab
 
       def has_local_branches?
         request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request)
+        response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout)
 
         response.value
       end

lib/tasks/gitlab/storage.rake

@@ -2,10 +2,10 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_legacy_storage.count
+      legacy_projects_count = Project.with_unmigrated_storage.count
 
       if legacy_projects_count == 0
-        puts 'There are no projects using legacy storage. Nothing to do!'
+        puts 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -23,22 +23,42 @@ namespace :gitlab do
 
     desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
     task legacy_projects: :environment do
-      projects_summary(Project.with_legacy_storage)
+      relation_summary('projects', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Legacy Storage'
     task list_legacy_projects: :environment do
-      projects_list(Project.with_legacy_storage)
+      projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
     task hashed_projects: :environment do
-      projects_summary(Project.with_hashed_storage)
+      relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Hashed Storage'
     task list_hashed_projects: :environment do
-      projects_list(Project.with_hashed_storage)
+      projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
+    task legacy_attachments: :environment do
+      relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
+    task list_legacy_attachments: :environment do
+      attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
+    task hashed_attachments: :environment do
+      relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
+    task list_hashed_attachments: :environment do
+      attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
     end
 
     def batch_size
@@ -46,29 +66,43 @@ namespace :gitlab do
     end
 
     def project_id_batches(&block)
-      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+      Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
         ids = relation.pluck(:id)
 
         yield ids.min, ids.max
       end
     end
 
-    def projects_summary(relation)
-      projects_count = relation.count
-      puts "* Found #{projects_count} projects".color(:green)
+    def legacy_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def hashed_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+        ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def relation_summary(relation_name, relation)
+      relation_count = relation.count
+      puts "* Found #{relation_count} #{relation_name}".color(:green)
 
-      projects_count
+      relation_count
     end
 
-    def projects_list(relation)
-      projects_count = projects_summary(relation)
+    def projects_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
 
       projects = relation.with_route
       limit = ENV.fetch('LIMIT', 500).to_i
 
-      return unless projects_count > 0
+      return unless relation_count > 0
 
-      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
 
       counter = 0
       projects.find_in_batches(batch_size: batch_size) do |batch|
@@ -81,5 +115,26 @@ namespace :gitlab do
         end
       end
     end
+
+    def attachments_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
+
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless relation_count > 0
+
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
+
+      counter = 0
+      relation.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |upload|
+          counter += 1
+
+          puts "  - #{upload.path} (id: #{upload.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
   end
 end

spec/ee/spec/features/epics/update_epic_spec.rb

@@ -3,7 +3,16 @@ require 'spec_helper'
 feature 'Update Epic', :js do
   let(:user) { create(:user) }
   let(:group) { create(:group, :public) }
-  let(:epic) { create(:epic, group: group) }
+
+  let(:markdown) do
+    <<-MARKDOWN.strip_heredoc
+    This is a task list:
+
+    - [ ] Incomplete entry 1
+    MARKDOWN
+  end
+
+  let(:epic) { create(:epic, group: group, description: markdown) }
 
   before do
     stub_licensed_features(epics: true)
@@ -51,6 +60,16 @@ feature 'Update Epic', :js do
       expect(page).not_to have_selector('.uploading-container .button-attach-file')
     end
 
+    it 'updates the tasklist' do
+      expect(page).to have_selector('ul.task-list',      count: 1)
+      expect(page).to have_selector('li.task-list-item', count: 1)
+      expect(page).to have_selector('ul input[checked]', count: 0)
+
+      find('.task-list .task-list-item', text: 'Incomplete entry 1').find('input').click
+
+      expect(page).to have_selector('ul input[checked]', count: 1)
+    end
+
     # Autocomplete is disabled for epics until #4084 is resolved
     describe 'autocomplete disabled' do
       it 'does not open atwho container' do

spec/ee/spec/finders/geo/project_registry_finder_spec.rb

+require 'spec_helper'
+
+# Disable transactions via :truncate method because a foreign table
+# can't see changes inside a transaction of a different connection.
+describe Geo::ProjectRegistryFinder, :geo, :truncate do
+  include ::EE::GeoHelpers
+
+  let(:secondary) { create(:geo_node) }
+  let(:synced_group) { create(:group) }
+  let!(:project_not_synced) { create(:project) }
+  let(:project_repository_dirty) { create(:project) }
+  let(:project_wiki_dirty) { create(:project) }
+
+  subject { described_class.new(current_node: secondary) }
+
+  before do
+    stub_current_geo_node(secondary)
+  end
+
+  context 'FDW' do
+    before do
+      skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
+    end
+
+    describe '#find_unsynced_projects' do
+      it 'delegates to #fdw_find_unsynced_projects' do
+        expect(subject).to receive(:fdw_find_unsynced_projects).and_call_original
+
+        subject.find_unsynced_projects(batch_size: 10)
+      end
+
+      it 'delegates to #legacy_find_unsynced_projects when node has selective sync' do
+        secondary.update_attribute(:namespaces, [synced_group])
+
+        expect(subject).to receive(:legacy_find_unsynced_projects).and_call_original
+
+        subject.find_unsynced_projects(batch_size: 10)
+      end
+
+      it 'returns projects without an entry on the tracking database' do
+        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+
+        projects = subject.find_unsynced_projects(batch_size: 10)
+
+        expect(projects.count).to eq(1)
+        expect(projects.first.id).to eq(project_not_synced.id)
+      end
+    end
+
+    describe '#find_projects_updated_recently' do
+      it 'delegates to #fdw_find_projects_updated_recently' do
+        expect(subject).to receive(:fdw_find_projects_updated_recently).and_call_original
+
+        subject.find_projects_updated_recently(batch_size: 10)
+      end
+
+      it 'delegates to #legacy_find_projects_updated_recently when node has selective sync' do
+        secondary.update_attribute(:namespaces, [synced_group])
+
+        expect(subject).to receive(:legacy_find_projects_updated_recently).and_call_original
+
+        subject.find_projects_updated_recently(batch_size: 10)
+      end
+
+      it 'returns projects with a dirty entry on the tracking database' do
+        project_repository_dirty = create(:project)
+        project_wiki_dirty = create(:project)
+
+        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+        create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
+
+        projects = subject.find_projects_updated_recently(batch_size: 10)
+
+        expect(projects.pluck(:id)).to match_array([project_repository_dirty.id, project_wiki_dirty.id])
+      end
+    end
+  end
+
+  context 'Legacy' do
+    before do
+      allow(Gitlab::Geo).to receive(:fdw?).and_return(false)
+    end
+
+    describe '#find_unsynced_projects' do
+      it 'delegates to #legacy_find_unsynced_projects' do
+        expect(subject).to receive(:legacy_find_unsynced_projects).and_call_original
+
+        subject.find_unsynced_projects(batch_size: 10)
+      end
+
+      it 'returns projects without an entry on the tracking database' do
+        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+
+        projects = subject.find_unsynced_projects(batch_size: 10)
+
+        expect(projects).to match_array([project_not_synced])
+      end
+    end
+
+    describe '#find_projects_updated_recently' do
+      it 'delegates to #legacy_find_projects_updated_recently' do
+        expect(subject).to receive(:legacy_find_projects_updated_recently).and_call_original
+
+        subject.find_projects_updated_recently(batch_size: 10)
+      end
+
+      it 'returns projects with a dirty entry on the tracking database' do
+        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+        create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
+
+        projects = subject.find_projects_updated_recently(batch_size: 10)
+
+        expect(projects.pluck(:id)).to match_array([project_repository_dirty.id, project_wiki_dirty.id])
+      end
+    end
+  end
+end

spec/ee/spec/models/ee/ci/variable_spec.rb

@@ -11,4 +11,22 @@ describe Ci::Variable do
     is_expected.to validate_uniqueness_of(:key)
       .scoped_to(:project_id, :environment_scope)
   end
+
+  describe '#environment_scope=' do
+    context 'when the new environment_scope is nil' do
+      it 'strips leading and trailing whitespaces' do
+        subject.environment_scope = nil
+
+        expect(subject.environment_scope).to eq('')
+      end
+    end
+
+    context 'when the new environment_scope has leadind and trailing whitespaces' do
+      it 'strips leading and trailing whitespaces' do
+        subject.environment_scope = ' * '
+
+        expect(subject.environment_scope).to eq('*')
+      end
+    end
+  end
 end

spec/ee/spec/services/ee/projects/hashed_storage/migrate_attachments_service_spec.rb

+require 'spec_helper'
+
+describe Projects::HashedStorage::MigrateAttachmentsService do
+  let(:project) { create(:project, storage_version: 1) }
+  let(:service) { described_class.new(project) }
+  let(:legacy_storage) { Storage::LegacyProject.new(project) }
+  let(:hashed_storage) { Storage::HashedProject.new(project) }
+  let(:old_attachments_path) { legacy_storage.disk_path }
+  let(:new_attachments_path) { hashed_storage.disk_path }
+
+  describe '#execute' do
+    set(:primary) { create(:geo_node, :primary) }
+    set(:secondary) { create(:geo_node) }
+
+    context 'on success' do
+      before do
+        FileUtils.mkdir_p(FileUploader.dynamic_path_builder(old_attachments_path))
+      end
+
+      it 'returns true' do
+        expect(service.execute).to be_truthy
+      end
+
+      it 'creates a Geo::HashedStorageAttachmentsEvent' do
+        expect { service.execute }.to change(Geo::EventLog, :count).by(1)
+
+        event = Geo::EventLog.first.event
+
+        expect(event).to be_a(Geo::HashedStorageAttachmentsEvent)
+        expect(event).to have_attributes(
+          old_attachments_path: old_attachments_path,
+          new_attachments_path: new_attachments_path
+        )
+      end
+    end
+
+    context 'on failure' do
+      it 'does not create a Geo event when skipped' do
+        expect { service.execute }.not_to change { Geo::EventLog.count }
+      end
+
+      it 'does not create a Geo event on failure' do
+        expect(service).to receive(:move_folder!).and_raise(::Projects::HashedStorage::AttachmentMigrationError)
+        expect { service.execute }.to raise_error(::Projects::HashedStorage::AttachmentMigrationError)
+        expect(Geo::EventLog.count).to eq(0)
+      end
+    end
+  end
+end