Merge branch 'master' into backstage/gb/rename-ci-cd-processing-sidekiq-queues

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

GITALY_SERVER_VERSION

-0.32.0
+0.33.0

Gemfile

@@ -152,7 +152,7 @@ gem 'acts-as-taggable-on', '~> 4.0'
 gem 'sidekiq', '~> 5.0'
 gem 'sidekiq-cron', '~> 0.6.0'
 gem 'redis-namespace', '~> 1.5.2'
-gem 'sidekiq-limit_fetch', '~> 3.4'
+gem 'sidekiq-limit_fetch', '~> 3.4', require: false
 
 # Cron Parser
 gem 'rufus-scheduler', '~> 3.4'

app/assets/javascripts/project_select_combo_button.js

@@ -4,10 +4,10 @@ export default class ProjectSelectComboButton {
   constructor(select) {
     this.projectSelectInput = $(select);
     this.newItemBtn = $('.new-project-item-link');
-    this.newItemBtnBaseText = this.newItemBtn.data('label');
-    this.itemType = this.deriveItemTypeFromLabel();
+    this.resourceType = this.newItemBtn.data('type');
+    this.resourceLabel = this.newItemBtn.data('label');
+    this.formattedText = this.deriveTextVariants();
     this.groupId = this.projectSelectInput.data('groupId');
-
     this.bindEvents();
     this.initLocalStorage();
   }
@@ -23,9 +23,7 @@ export default class ProjectSelectComboButton {
     const localStorageIsSafe = AccessorUtilities.isLocalStorageAccessSafe();
 
     if (localStorageIsSafe) {
-      const itemTypeKebabed = this.newItemBtnBaseText.toLowerCase().split(' ').join('-');
-
-      this.localStorageKey = ['group', this.groupId, itemTypeKebabed, 'recent-project'].join('-');
+      this.localStorageKey = ['group', this.groupId, this.formattedText.localStorageItemType, 'recent-project'].join('-');
       this.setBtnTextFromLocalStorage();
     }
   }
@@ -57,19 +55,14 @@ export default class ProjectSelectComboButton {
   setNewItemBtnAttributes(project) {
     if (project) {
       this.newItemBtn.attr('href', project.url);
-      this.newItemBtn.text(`${this.newItemBtnBaseText} in ${project.name}`);
+      this.newItemBtn.text(`${this.formattedText.defaultTextPrefix} in ${project.name}`);
       this.newItemBtn.enable();
     } else {
-      this.newItemBtn.text(`Select project to create ${this.itemType}`);
+      this.newItemBtn.text(`Select project to create ${this.formattedText.presetTextSuffix}`);
       this.newItemBtn.disable();
     }
   }
 
-  deriveItemTypeFromLabel() {
-    // label is either 'New issue' or 'New merge request'
-    return this.newItemBtnBaseText.split(' ').slice(1).join(' ');
-  }
-
   getProjectFromLocalStorage() {
     const projectString = localStorage.getItem(this.localStorageKey);
 
@@ -81,5 +74,19 @@ export default class ProjectSelectComboButton {
 
     localStorage.setItem(this.localStorageKey, projectString);
   }
+
+  deriveTextVariants() {
+    const defaultTextPrefix = this.resourceLabel;
+
+    // the trailing slice call depluralizes each of these strings (e.g. new-issues -> new-issue)
+    const localStorageItemType = `new-${this.resourceType.split('_').join('-').slice(0, -1)}`;
+    const presetTextSuffix = this.resourceType.split('_').join(' ').slice(0, -1);
+
+    return {
+      localStorageItemType, // new-issue / new-merge-request
+      defaultTextPrefix, // New issue / New merge request
+      presetTextSuffix, // issue / merge request
+    };
+  }
 }
 

app/assets/stylesheets/framework/dropdowns.scss

@@ -761,7 +761,7 @@
         &:hover,
         &:active,
         &:focus {
-          background-color: $gray-darker;
+          background-color: $dropdown-item-hover-bg;
           color: $gl-text-color;
         }
 

app/assets/stylesheets/framework/selects.scss

@@ -264,3 +264,41 @@
 .ajax-users-dropdown {
   min-width: 250px !important;
 }
+
+// TODO: change global style
+.ajax-project-dropdown {
+  &.select2-drop {
+    color: $gl-text-color;
+  }
+
+  .select2-results {
+    .select2-no-results,
+    .select2-searching,
+    .select2-ajax-error,
+    .select2-selection-limit {
+      background: transparent;
+    }
+
+    .select2-result {
+      padding: 0 1px;
+
+      .select2-match {
+        font-weight: bold;
+        text-decoration: none;
+      }
+
+      .select2-result-label {
+        padding: #{$gl-padding / 2} $gl-padding;
+      }
+
+      &.select2-highlighted {
+        background-color: transparent !important;
+        color: $gl-text-color;
+
+        .select2-result-label {
+          background-color: $dropdown-item-hover-bg;
+        }
+      }
+    }
+  }
+}

app/assets/stylesheets/framework/variables.scss

@@ -294,7 +294,7 @@ $dropdown-input-focus-shadow: rgba($dropdown-input-focus-border, .4);
 $dropdown-loading-bg: rgba(#fff, .6);
 $dropdown-chevron-size: 10px;
 $dropdown-toggle-active-border-color: darken($border-color, 14%);
-
+$dropdown-item-hover-bg: $gray-darker;
 
 /*
 * Filtered Search

app/controllers/groups_controller.rb

@@ -26,6 +26,13 @@ class GroupsController < Groups::ApplicationController
 
   def new
     @group = Group.new
+
+    if params[:parent_id].present?
+      parent = Group.find_by(id: params[:parent_id])
+      if can?(current_user, :create_subgroup, parent)
+        @group.parent = parent
+      end
+    end
   end
 
   def create

app/helpers/application_settings_helper.rb

@@ -116,6 +116,7 @@ module ApplicationSettingsHelper
       :email_author_in_body,
       :enabled_git_access_protocol,
       :gravatar_enabled,
+      :hashed_storage_enabled,
       :help_page_hide_commercial_content,
       :help_page_support_url,
       :help_page_text,

app/helpers/events_helper.rb

@@ -176,7 +176,7 @@ module EventsHelper
     sanitize(
       text,
       tags: %w(a img gl-emoji b pre code p span),
-      attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-name', 'data-unicode-version']
+      attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-src', 'data-name', 'data-unicode-version']
     )
   end
 

app/models/ci/pipeline_schedule.rb

@@ -9,7 +9,7 @@ module Ci
     belongs_to :owner, class_name: 'User'
     has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline'
     has_many :pipelines
-    has_many :variables, class_name: 'Ci::PipelineScheduleVariable'
+    has_many :variables, class_name: 'Ci::PipelineScheduleVariable', validate: false
 
     validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? }
     validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? }

app/models/ci/pipeline_schedule_variable.rb

@@ -4,5 +4,7 @@ module Ci
     include HasVariable
 
     belongs_to :pipeline_schedule
+
+    validates :key, uniqueness: { scope: :pipeline_schedule_id }
   end
 end

app/models/ci/stage.rb

 module Ci
   class Stage < ActiveRecord::Base
     extend Ci::Model
+    include Importable
+    include HasStatus
+    include Gitlab::OptimisticLocking
+
+    enum status: HasStatus::STATUSES_ENUM
 
     belongs_to :project
     belongs_to :pipeline
 
-    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
-    has_many :builds, foreign_key: :commit_id
+    has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
+    has_many :builds, foreign_key: :stage_id
+
+    validates :project, presence: true, unless: :importing?
+    validates :pipeline, presence: true, unless: :importing?
+    validates :name, presence: true, unless: :importing?
+
+    state_machine :status, initial: :created do
+      event :enqueue do
+        transition created: :pending
+        transition [:success, :failed, :canceled, :skipped] => :running
+      end
+
+      event :run do
+        transition any - [:running] => :running
+      end
+
+      event :skip do
+        transition any - [:skipped] => :skipped
+      end
+
+      event :drop do
+        transition any - [:failed] => :failed
+      end
+
+      event :succeed do
+        transition any - [:success] => :success
+      end
+
+      event :cancel do
+        transition any - [:canceled] => :canceled
+      end
+
+      event :block do
+        transition any - [:manual] => :manual
+      end
+    end
+
+    def update_status
+      retry_optimistic_lock(self) do
+        case statuses.latest.status
+        when 'pending' then enqueue
+        when 'running' then run
+        when 'success' then succeed
+        when 'failed' then drop
+        when 'canceled' then cancel
+        when 'manual' then block
+        when 'skipped' then skip
+        else skip
+        end
+      end
+    end
   end
 end

app/models/commit_status.rb

@@ -39,14 +39,14 @@ class CommitStatus < ActiveRecord::Base
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
 
   state_machine :status do
-    event :enqueue do
-      transition [:created, :skipped, :manual] => :pending
-    end
-
     event :process do
       transition [:skipped, :manual] => :created
     end
 
+    event :enqueue do
+      transition [:created, :skipped, :manual] => :pending
+    end
+
     event :run do
       transition pending: :running
     end
@@ -91,6 +91,7 @@ class CommitStatus < ActiveRecord::Base
           end
         end
 
+        StageUpdateWorker.perform_async(commit_status.stage_id)
         ExpireJobCacheWorker.perform_async(commit_status.id)
       end
     end

app/models/concerns/has_status.rb

@@ -8,6 +8,8 @@ module HasStatus
   ACTIVE_STATUSES = %w[pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
   ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze
+  STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
+                    failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
 
   class_methods do
     def status_sql

app/models/concerns/storage/legacy_project.rb

-module Storage
-  module LegacyProject
-    extend ActiveSupport::Concern
-
-    def disk_path
-      full_path
-    end
-
-    def ensure_storage_path_exist
-      gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
-    end
-
-    def rename_repo
-      path_was = previous_changes['path'].first
-      old_path_with_namespace = File.join(namespace.full_path, path_was)
-      new_path_with_namespace = File.join(namespace.full_path, path)
-
-      Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      if has_container_registry_tags?
-        Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
-
-        # we currently doesn't support renaming repository if it contains images in container registry
-        raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
-      end
-
-      expire_caches_before_rename(old_path_with_namespace)
-
-      if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
-        # If repository moved successfully we need to send update instructions to users.
-        # However we cannot allow rollback since we moved repository
-        # So we basically we mute exceptions in next actions
-        begin
-          gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
-          send_move_instructions(old_path_with_namespace)
-          expires_full_path_cache
-
-          @old_path_with_namespace = old_path_with_namespace
-
-          SystemHooksService.new.execute_hooks_for(self, :rename)
-
-          @repository = nil
-        rescue => e
-          Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
-          # Returning false does not rollback after_* transaction but gives
-          # us information about failing some of tasks
-          false
-        end
-      else
-        Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-        # if we cannot move namespace directory we should rollback
-        # db changes in order to prevent out of sync between db and fs
-        raise StandardError.new('repository cannot be renamed')
-      end
-
-      Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
-      Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
-    end
-
-    def create_repository(force: false)
-      # Forked import is handled asynchronously
-      return if forked? && !force
-
-      if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
-        repository.after_create
-        true
-      else
-        errors.add(:base, 'Failed to create repository via gitlab-shell')
-        false
-      end
-    end
-  end
-end

app/models/event.rb

@@ -83,6 +83,10 @@ class Event < ActiveRecord::Base
   self.inheritance_column = 'action'
 
   class << self
+    def model_name
+      ActiveModel::Name.new(self, nil, 'event')
+    end
+
     def find_sti_class(action)
       if action.to_i == PUSHED
         PushEvent
@@ -438,6 +442,12 @@ class Event < ActiveRecord::Base
     EventForMigration.create!(new_attributes)
   end
 
+  def to_partial_path
+    # We are intentionally using `Event` rather than `self.class` so that
+    # subclasses also use the `Event` implementation.
+    Event._to_partial_path
+  end
+
   private
 
   def recent_update?

app/models/issue.rb

@@ -9,11 +9,8 @@ class Issue < ActiveRecord::Base
   include Spammable
   include FasterCacheKeys
   include RelativePositioning
-  include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
-
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
   AnyDueDate    = DueDateStruct.new('Any Due Date', '').freeze

app/models/merge_request.rb

@@ -7,7 +7,6 @@ class MergeRequest < ActiveRecord::Base
   include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
   ignore_column :locked_at
 
   belongs_to :target_project, class_name: "Project"

app/models/project.rb

@@ -17,7 +17,6 @@ class Project < ActiveRecord::Base
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
   include Routable
-  include Storage::LegacyProject
 
   extend Gitlab::ConfigHelper
 
@@ -25,6 +24,7 @@ class Project < ActiveRecord::Base
 
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
+  LATEST_STORAGE_VERSION = 1
 
   cache_markdown_field :description, pipeline: :description
 
@@ -32,6 +32,8 @@ class Project < ActiveRecord::Base
            :merge_requests_enabled?, :issues_enabled?, to: :project_feature,
                                                        allow_nil: true
 
+  delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
+
   default_value_for :archived, false
   default_value_for :visibility_level, gitlab_config_features.visibility_level
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
@@ -44,32 +46,24 @@ class Project < ActiveRecord::Base
   default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
-  after_create :ensure_storage_path_exist
-  after_create :create_project_feature, unless: :project_feature
-  after_save :update_project_statistics, if: :namespace_id_changed?
+  add_authentication_token_field :runners_token
+  before_save :ensure_runners_token
 
-  # set last_activity_at to the same as created_at
+  after_save :update_project_statistics, if: :namespace_id_changed?
+  after_create :create_project_feature, unless: :project_feature
   after_create :set_last_activity_at
-  def set_last_activity_at
-    update_column(:last_activity_at, self.created_at)
-  end
-
   after_create :set_last_repository_updated_at
-  def set_last_repository_updated_at
-    update_column(:last_repository_updated_at, self.created_at)
-  end
+  after_update :update_forks_visibility_level
 
   before_destroy :remove_private_deploy_keys
   after_destroy -> { run_after_commit { remove_pages } }
 
-  # update visibility_level of forks
-  after_update :update_forks_visibility_level
-
   after_validation :check_pending_delete
 
-  # Legacy Storage specific hooks
-
-  after_save :ensure_storage_path_exist, if: :namespace_id_changed?
+  # Storage specific hooks
+  after_initialize :use_hashed_storage
+  after_create :ensure_storage_path_exists
+  after_save :ensure_storage_path_exists, if: :namespace_id_changed?
 
   acts_as_taggable
 
@@ -238,9 +232,6 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
 
-  add_authentication_token_field :runners_token
-  before_save :ensure_runners_token
-
   mount_uploader :avatar, AvatarUploader
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
@@ -487,6 +478,10 @@ class Project < ActiveRecord::Base
     @repository ||= Repository.new(full_path, self, disk_path: disk_path)
   end
 
+  def reload_repository!
+    @repository = nil
+  end
+
   def container_registry_url
     if Gitlab.config.registry.enabled
       "#{Gitlab.config.registry.host_port}/#{full_path.downcase}"
@@ -1004,6 +999,19 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def create_repository(force: false)
+    # Forked import is handled asynchronously
+    return if forked? && !force
+
+    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+      repository.after_create
+      true
+    else
+      errors.add(:base, 'Failed to create repository via gitlab-shell')
+      false
+    end
+  end
+
   def hook_attrs(backward: true)
     attrs = {
       name: name,
@@ -1086,6 +1094,7 @@ class Project < ActiveRecord::Base
     !!repository.exists?
   end
 
+  # update visibility_level of forks
   def update_forks_visibility_level
     return unless visibility_level < visibility_level_was
 
@@ -1213,7 +1222,8 @@ class Project < ActiveRecord::Base
   end
 
   def pages_path
-    File.join(Settings.pages.path, disk_path)
+    # TODO: when we migrate Pages to work with new storage types, change here to use disk_path
+    File.join(Settings.pages.path, full_path)
   end
 
   def public_pages_path
@@ -1252,6 +1262,50 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def rename_repo
+    new_full_path = build_full_path
+
+    Rails.logger.error "Attempting to rename #{full_path_was} -> #{new_full_path}"
+
+    if has_container_registry_tags?
+      Rails.logger.error "Project #{full_path_was} cannot be renamed because container registry tags are present!"
+
+      # we currently doesn't support renaming repository if it contains images in container registry
+      raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
+    end
+
+    expire_caches_before_rename(full_path_was)
+
+    if storage.rename_repo
+      Gitlab::AppLogger.info "Project was renamed: #{full_path_was} -> #{new_full_path}"
+      rename_repo_notify!
+      after_rename_repo
+    else
+      Rails.logger.error "Repository could not be renamed: #{full_path_was} -> #{new_full_path}"
+
+      # if we cannot move namespace directory we should rollback
+      # db changes in order to prevent out of sync between db and fs
+      raise StandardError.new('repository cannot be renamed')
+    end
+  end
+
+  def rename_repo_notify!
+    send_move_instructions(full_path_was)
+    expires_full_path_cache
+
+    self.old_path_with_namespace = full_path_was
+    SystemHooksService.new.execute_hooks_for(self, :rename)
+
+    reload_repository!
+  end
+
+  def after_rename_repo
+    path_before_change = previous_changes['path'].first
+
+    Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+  end
+
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1410,6 +1464,10 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def full_path_was
+    File.join(namespace.full_path, previous_changes['path'].first)
+  end
+
   alias_method :name_with_namespace, :full_name
   alias_method :human_name, :full_name
   # @deprecated cannot remove yet because it has an index with its name in elasticsearch
@@ -1419,8 +1477,36 @@ class Project < ActiveRecord::Base
     Projects::ForksCountService.new(self).count
   end
 
+  def legacy_storage?
+    self.storage_version.nil?
+  end
+
   private
 
+  def storage
+    @storage ||=
+      if self.storage_version && self.storage_version >= 1
+        Storage::HashedProject.new(self)
+      else
+        Storage::LegacyProject.new(self)
+      end
+  end
+
+  def use_hashed_storage
+    if self.new_record? && current_application_settings.hashed_storage_enabled
+      self.storage_version = LATEST_STORAGE_VERSION
+    end
+  end
+
+  # set last_activity_at to the same as created_at
+  def set_last_activity_at
+    update_column(:last_activity_at, self.created_at)
+  end
+
+  def set_last_repository_updated_at
+    update_column(:last_repository_updated_at, self.created_at)
+  end
+
   def cross_namespace_reference?(from)
     case from
     when Project

app/models/storage/hashed_project.rb

+module Storage
+  class HashedProject
+    attr_accessor :project
+    delegate :gitlab_shell, :repository_storage_path, to: :project
+
+    ROOT_PATH_PREFIX = '@hashed'.freeze
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      "#{base_dir}/#{disk_hash}" if disk_hash
+    end
+
+    def ensure_storage_path_exists
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      true
+    end
+
+    private
+
+    # Generates the hash for the project path and name on disk
+    # If you need to refer to the repository on disk, use the `#disk_path`
+    def disk_hash
+      @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
+    end
+  end
+end

app/models/storage/legacy_project.rb

+module Storage
+  class LegacyProject
+    attr_accessor :project
+    delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      namespace.full_path
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      project.full_path
+    end
+
+    def ensure_storage_path_exists
+      return unless namespace
+
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      new_full_path = project.build_full_path
+
+      if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
+        # If repository moved successfully we need to send update instructions to users.
+        # However we cannot allow rollback since we moved repository
+        # So we basically we mute exceptions in next actions
+        begin
+          gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
+          return true
+        rescue => e
+          Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"
+          # Returning false does not rollback after_* transaction but gives
+          # us information about failing some of tasks
+          return false
+        end
+      end
+
+      false
+    end
+  end
+end

app/policies/group_policy.rb

@@ -13,6 +13,8 @@ class GroupPolicy < BasePolicy
   condition(:master) { access_level >= GroupMember::MASTER }
   condition(:reporter) { access_level >= GroupMember::REPORTER }
 
+  condition(:nested_groups_supported, scope: :global) { Group.supports_nested_groups? }
+
   condition(:has_projects) do
     GroupProjectsFinder.new(group: @subject, current_user: @user).execute.any?
   end
@@ -42,7 +44,7 @@ class GroupPolicy < BasePolicy
     enable :change_visibility_level
   end
 
-  rule { owner & can_create_group }.enable :create_subgroup
+  rule { owner & can_create_group & nested_groups_supported }.enable :create_subgroup
 
   rule { public_group | logged_in_viewable }.enable :view_globally
 

app/services/ci/create_pipeline_service.rb

@@ -176,9 +176,14 @@ module Ci
     end
 
     def error(message, save: false)
-      pipeline.errors.add(:base, message)
-      pipeline.drop if save
-      pipeline
+      pipeline.tap do
+        pipeline.errors.add(:base, message)
+
+        if save
+          pipeline.drop
+          update_merge_requests_head_pipeline
+        end
+      end
     end
 
     def pipeline_created_counter

app/services/groups/create_service.rb

@@ -13,9 +13,9 @@ module Groups
         return @group
       end
 
-      if @group.parent && !can?(current_user, :admin_group, @group.parent)
+      if @group.parent && !can?(current_user, :create_subgroup, @group.parent)
         @group.parent = nil
-        @group.errors.add(:parent_id, 'manage access required to create subgroup')
+        @group.errors.add(:parent_id, 'You don’t have permission to create a subgroup in this group.')
 
         return @group
       end

app/services/groups/destroy_service.rb

@@ -13,7 +13,7 @@ module Groups
         # Execute the destruction of the models immediately to ensure atomic cleanup.
         # Skip repository removal because we remove directory with namespace
         # that contain all these repositories
-        ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute
+        ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute
       end
 
       group.children.each do |group|

app/services/merge_requests/create_from_issue_service.rb

@@ -3,6 +3,8 @@ module MergeRequests
     def execute
       return error('Invalid issue iid') unless issue_iid.present? && issue.present?
 
+      params[:label_ids] = issue.label_ids if issue.label_ids.any?
+
       result = CreateBranchService.new(project, current_user).execute(branch_name, ref)
       return result if result[:status] == :error
 
@@ -43,7 +45,8 @@ module MergeRequests
       {
         source_project_id: project.id,
         source_branch: branch_name,
-        target_project_id: project.id
+        target_project_id: project.id,
+        milestone_id: issue.milestone_id
       }
     end
 

app/services/users/destroy_service.rb

@@ -35,16 +35,18 @@ module Users
         Groups::DestroyService.new(group, current_user).execute
       end
 
+      namespace = user.namespace
+      namespace.prepare_for_destroy
+
       user.personal_projects.each do |project|
         # Skip repository removal because we remove directory with namespace
         # that contain all this repositories
-        ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute
+        ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute
       end
 
       MigrateToGhostUserService.new(user).execute unless options[:hard_delete]
 
       # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing
-      namespace = user.namespace
       user_data = user.destroy
       namespace.really_destroy!
 

app/views/admin/application_settings/_form.html.haml

@@ -362,7 +362,9 @@
   %fieldset
     %legend Background Jobs
     %p
-      These settings require a restart to take effect.
+      These settings require a
+      = link_to 'restart', help_page_path('administration/restart_gitlab')
+      to take effect.
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox
@@ -490,6 +492,16 @@
 
   %fieldset
     %legend Repository Storage
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :hashed_storage_enabled do
+            = f.check_box :hashed_storage_enabled
+            Create new projects using hashed storage paths
+          .help-block
+            Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents
+            repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance.
+            %em (EXPERIMENTAL)
     .form-group
       = f.label :repository_storages, 'Storage paths for new projects', class: 'control-label col-sm-2'
       .col-sm-10
@@ -499,6 +511,7 @@
           = succeed "." do
             = link_to "repository storages documentation", help_page_path("administration/repository_storages")
 
+
   %fieldset
     %legend Repository Checks
     .form-group

app/views/dashboard/issues.html.haml

@@ -8,14 +8,14 @@
   - content_for :breadcrumbs_extra do
     = link_to params.merge(rss_url_options), class: 'btn has-tooltip append-right-10', title: 'Subscribe' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues'
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues
 
 .top-area
   = render 'shared/issuable/nav', type: :issues
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
     = link_to params.merge(rss_url_options), class: 'btn has-tooltip', title: 'Subscribe' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues'
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues
 
 = render 'shared/issuable/filter', type: :issues
 = render 'shared/issues'

app/views/dashboard/merge_requests.html.haml

@@ -4,12 +4,12 @@
 
 - if show_new_nav?
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests'
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests
 
 .top-area
   = render 'shared/issuable/nav', type: :merge_requests
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests'
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests
 
 = render 'shared/issuable/filter', type: :merge_requests
 = render 'shared/merge_requests'

app/views/dashboard/milestones/index.html.haml

@@ -4,13 +4,13 @@
 
 - if show_new_nav?
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true
+    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones
 
 .top-area
   = render 'shared/milestones_filter', counts: @milestone_states
 
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true
+    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones
 
 .milestones
   %ul.content-list

app/views/groups/issues.html.haml

@@ -12,7 +12,7 @@
   - content_for :breadcrumbs_extra do
     = link_to params.merge(rss_url_options), class: 'btn btn-default append-right-10' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue"
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues
 
 - if group_issues_exists
   .top-area
@@ -22,7 +22,7 @@
         = icon('rss')
         %span.icon-label
           Subscribe
-      = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue"
+      = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues
 
   = render 'shared/issuable/search_bar', type: :issues
 

app/views/groups/merge_requests.html.haml

@@ -2,7 +2,7 @@
 
 - if show_new_nav? && current_user
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request"
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request",  type: :merge_requests
 
 - if @group_merge_requests.empty?
   = render 'shared/empty_states/merge_requests', project_select_button: true
@@ -11,7 +11,7 @@
     = render 'shared/issuable/nav', type: :merge_requests
     - if current_user
       .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-        = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request"
+        = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request",  type: :merge_requests
 
   = render 'shared/issuable/filter', type: :merge_requests
 

app/views/shared/_group_form.html.haml

 - content_for :page_specific_javascripts do
   = page_specific_javascript_bundle_tag('group')
-- parent = GroupFinder.new(current_user).execute(id: params[:parent_id] || @group.parent_id)
+- parent = @group.parent
 - group_path = root_url
 - group_path << parent.full_path + '/' if parent
 
@@ -13,13 +13,12 @@
         %span>= root_url
         - if parent
           %strong= parent.full_path + '/'
+      = f.hidden_field :parent_id
       = f.text_field :path, placeholder: 'open-source', class: 'form-control',
         autofocus: local_assigns[:autofocus] || false, required: true,
         pattern: Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX_JS,
         title: 'Please choose a group path with no special characters.',
         "data-bind-in" => "#{'create_chat_team' if Gitlab.config.mattermost.enabled}"
-      - if parent
-        = f.hidden_field :parent_id, value: parent.id
 
     - if @group.persisted?
       .alert.alert-warning.prepend-top-10

app/views/shared/_new_project_item_select.html.haml

 - if any_projects?(@projects)
   .project-item-select-holder.btn-group.pull-right
-    %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label] } }
+    %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label], type: local_assigns[:type] } }
       = icon('spinner spin')
     = project_select_tag :project_path, class: "project-item-select", data: { include_groups: local_assigns[:include_groups], order_by: 'last_activity_at', relative_path: local_assigns[:path] }, with_feature_enabled: local_assigns[:with_feature_enabled]
     %button.btn.btn-new.new-project-item-select-button

app/views/shared/empty_states/_issues.html.haml

@@ -15,7 +15,7 @@
           Issues can be bugs, tasks or ideas to be discussed.
           Also, issues are searchable and filterable.
         - if project_select_button
-          = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue'
+          = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue', type: :issues
         - else
           = link_to 'New issue', button_path, class: 'btn btn-new', title: 'New issue', id: 'new_issue_link'
       - else

app/views/shared/empty_states/_merge_requests.html.haml

@@ -14,7 +14,7 @@
         %p
           Interested parties can even contribute by pushing commits if they want to.
         - if project_select_button
-          = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request'
+          = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request', type: :merge_requests
         - else
           = link_to 'New merge request', button_path, class: 'btn btn-new', title: 'New merge request', id: 'new_merge_request_link'
       - else

app/workers/authorized_projects_worker.rb

@@ -4,18 +4,25 @@ class AuthorizedProjectsWorker
 
   # Schedules multiple jobs and waits for them to be completed.
   def self.bulk_perform_and_wait(args_list)
-    job_ids = bulk_perform_async(args_list)
+    waiter = Gitlab::JobWaiter.new(args_list.size)
 
-    Gitlab::JobWaiter.new(job_ids).wait
+    # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]]
+    # into [[1, "key"], [2, "key"], [3, "key"]]
+    waiting_args_list = args_list.map { |args| args << waiter.key }
+    bulk_perform_async(waiting_args_list)
+
+    waiter.wait
   end
 
   def self.bulk_perform_async(args_list)
     Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
   end
 
-  def perform(user_id)
+  def perform(user_id, notify_key = nil)
     user = User.find_by(id: user_id)
 
     user&.refresh_authorized_projects
+  ensure
+    Gitlab::JobWaiter.notify(notify_key, jid) if notify_key
   end
 end

app/workers/stage_update_worker.rb

+class StageUpdateWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(stage_id)
+    Ci::Stage.find_by(id: stage_id).try do |stage|
+      stage.update_status
+    end
+  end
+end

changelogs/unreleased/28283-uuid-storage.yml

+---
+title: Hashed Storage support for Repositories (EXPERIMENTAL)
+merge_request: 13246
+author:

changelogs/unreleased/35343-inherit-milestones-and-labels.yml

+---
+title: inherits milestone and labels when a merge request is created from issue
+merge_request: 13461 
+author: haseebeqx
+type: added

changelogs/unreleased/35845-improve-subgroup-creation-permissions.yml

+---
+title: Improves subgroup creation permissions
+merge_request: 13418
+author:
+type: bugifx

changelogs/unreleased/fix-gb-fix-head-pipeline-when-pipeline-has-errors.yml

+---
+title: Fix merge request pipeline status when pipeline has errors
+merge_request: 13664
+author:
+type: fixed

changelogs/unreleased/only-limit-fetch-when-requested.yml

+---
+title: Only require Sidekiq throttling library when enabled, to reduce cache misses
+merge_request:
+author:
+type: fixed

changelogs/unreleased/zj-remove-ci-api-v1.yml

+---
+title: Remove CI API v1
+merge_request:
+author:
+type: removed

config/routes/ci.rb

 namespace :ci do
-  # CI API
-  Ci::API::API.logger Rails.logger
-  mount Ci::API::API => '/api'
-
   resource :lint, only: [:show, :create]
 
   root to: redirect('/')

db/migrate/20170711145320_add_status_to_ci_stages.rb

+class AddStatusToCiStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_stages, :status, :integer
+  end
+end

db/migrate/20170720111708_add_lock_version_to_ci_stages.rb

+class AddLockVersionToCiStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_stages, :lock_version, :integer
+  end
+end

db/migrate/20170802013652_add_storage_fields_to_project.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddStorageFieldsToProject < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    add_column :projects, :storage_version, :integer, limit: 2
+  end
+
+  def down
+    remove_column :projects, :storage_version
+  end
+end

db/migrate/20170807071105_add_hashed_storage_to_settings.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddHashedStorageToSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :hashed_storage_enabled, :boolean, default: false
+  end
+
+  def down
+    remove_columns :application_settings, :hashed_storage_enabled
+  end
+end

db/post_migrate/20170711145558_migrate_stages_statuses.rb

+class MigrateStagesStatuses < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  BATCH_SIZE = 10000
+  RANGE_SIZE = 1000
+  MIGRATION = 'MigrateStageStatus'.freeze
+
+  class Stage < ActiveRecord::Base
+    self.table_name = 'ci_stages'
+    include ::EachBatch
+  end
+
+  def up
+    Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index|
+      relation.each_batch(of: RANGE_SIZE) do |batch|
+        range = relation.pluck('MIN(id)', 'MAX(id)').first
+        schedule = index * 5.minutes
+
+        BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range)
+      end
+    end
+  end
+
+  def down
+    disable_statement_timeout
+
+    update_column_in_batches(:ci_stages, :status, nil)
+  end
+end

db/schema.rb

@@ -128,6 +128,7 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.integer "performance_bar_allowed_group_id"
     t.boolean "password_authentication_enabled"
     t.boolean "project_export_enabled", default: true, null: false
+    t.boolean "hashed_storage_enabled", default: false, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -379,6 +380,8 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.string "name"
+    t.integer "status"
+    t.integer "lock_version"
   end
 
   add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
@@ -1206,6 +1209,7 @@ ActiveRecord::Schema.define(version: 20170822101017) do
     t.datetime "last_repository_updated_at"
     t.string "ci_config_path"
     t.text "delete_error"
+    t.integer "storage_version", limit: 2
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree

doc/api/README.md

@@ -55,15 +55,10 @@ following locations:
 - [Tags](tags.md)
 - [Todos](todos.md)
 - [Users](users.md)
-- [Validate CI configuration](ci/lint.md)
+- [Validate CI configuration](lint.md)
 - [V3 to V4](v3_to_v4.md)
 - [Version](version.md)
 
-The following documentation is for the [internal CI API](ci/README.md):
-
-- [Builds](ci/builds.md)
-- [Runners](ci/runners.md)
-
 ## Road to GraphQL
 
 Going forward, we will start on moving to

doc/api/ci/README.md

-# GitLab CI API
-
-## Purpose
-
-The main purpose of GitLab CI API is to provide the necessary data and context
-for GitLab CI Runners.
-
-All relevant information about the consumer API can be found in a
-[separate document](../../api/README.md).
-
-## API Prefix
-
-The current CI API prefix is `/ci/api/v1`.
-
-You need to prepend this prefix to all examples in this documentation, like:
-
-```bash
-GET /ci/api/v1/builds/:id/artifacts
-```
-
-## Resources
-
-- [Builds](builds.md)
-- [Runners](runners.md)

doc/api/ci/builds.md

-# Builds API
-
-API used by runners to receive and update builds.
-
->**Note:**
-This API is intended to be used only by Runners as their own
-communication channel. For the consumer API see the
-[Jobs API](../jobs.md).
-
-## Authentication
-
-This API uses two types of authentication:
-
-1. Unique Runner's token which is the token assigned to the Runner after it
-   has been registered.
-
-2. Using the build authorization token.
-   This is project's CI token that can be found under the **Builds** section of
-   a project's settings. The build authorization token can be passed as a
-   parameter or a value of `BUILD-TOKEN` header.
-
-These two methods of authentication are interchangeable.
-
-## Builds
-
-### Runs oldest pending build by runner
-
-```
-POST /ci/api/v1/builds/register
-```
-
-| Attribute | Type    | Required | Description         |
-|-----------|---------|----------|---------------------|
-| `token`   | string  | yes      | Unique runner token |
-
-
-```
-curl --request POST "https://gitlab.example.com/ci/api/v1/builds/register" --form "token=t0k3n"
-```
-
-**Responses:**
-
-| Status | Data |Description                                                                |
-|--------|------|---------------------------------------------------------------------------|
-| `201`  | yes  | When a build is scheduled for a runner                                    |
-| `204`  | no   | When no builds are scheduled for a runner (for GitLab Runner >= `v1.3.0`) |
-| `403`  | no   | When invalid token is used or no token is sent                            |
-| `404`  | no   | When no builds are scheduled for a runner (for GitLab Runner < `v1.3.0`) **or** when the runner is set to `paused` in GitLab runner's configuration page |
-
-### Update details of an existing build
-
-```
-PUT /ci/api/v1/builds/:id
-```
-
-| Attribute | Type    | Required | Description          |
-|-----------|---------|----------|----------------------|
-| `id`      | integer | yes      | The ID of a project  |
-| `token`   | string  | yes      | Unique runner token  |
-| `state`   | string  | no       | The state of a build |
-| `trace`   | string  | no       | The trace of a build |
-
-```
-curl --request PUT "https://gitlab.example.com/ci/api/v1/builds/1234" --form "token=t0k3n" --form "state=running" --form "trace=Running git clone...\n"
-```
-
-### Incremental build trace update
-
-Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header
-with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part
-must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416
-Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length.
-
-For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...`
-header and a trace part covered by this range.
-
-For a valid update API will return `202` response with:
-* `Build-Status: {status}` header containing current status of the build,
-* `Range: 0-{length}` header with the current trace length.
-
-```
-PATCH /ci/api/v1/builds/:id/trace.txt
-```
-
-Parameters:
-
-| Attribute | Type    | Required | Description          |
-|-----------|---------|----------|----------------------|
-| `id`      | integer | yes      | The ID of a build    |
-
-Headers:
-
-| Attribute       | Type    | Required | Description                       |
-|-----------------|---------|----------|-----------------------------------|
-| `BUILD-TOKEN`   | string  | yes      | The build authorization token     |
-| `Content-Range` | string  | yes      | Bytes range of trace that is sent |
-
-```
-curl --request PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" --header "BUILD-TOKEN=build_t0k3n" --header "Content-Range=0-21" --data "Running git clone...\n"
-```
-
-
-### Upload artifacts to build
-
-```
-POST /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| `id`      | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-| `file`    | mixed   | yes      | Artifacts file                |
-
-```
-curl --request POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n" --form "file=@/path/to/file"
-```
-
-### Download the artifacts file from build
-
-```
-GET /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| `id`      | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-
-```
-curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n"
-```
-
-### Remove the artifacts file from build
-
-```
-DELETE /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| ` id`     | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-
-```
-curl --request DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" --form "token=build_t0k3n"
-```

doc/api/ci/runners.md

-# Register and Delete Runners API
-
-API used by Runners to register and delete themselves.
-
->**Note:**
-This API is intended to be used only by Runners as their own
-communication channel. For the consumer API see the
-[new Runners API](../runners.md).
-
-## Authentication
-
-This API uses two types of authentication:
-
-1. Unique Runner's token, which is the token assigned to the Runner after it
-   has been registered.  This token can be found on the Runner's edit page (go to
-   **Project > Runners**, select one of the Runners listed under **Runners activated for
-   this project**).
-
-2. Using Runners' registration token.
-   This is a token that can be found in project's settings.
-   It can also be found in the **Admin > Runners** settings area.
-   There are two types of tokens you can pass: shared Runner registration
-   token or project specific registration token.
-
-## Register a new runner
-
-Used to make GitLab CI aware of available runners.
-
-```sh
-POST /ci/api/v1/runners/register
-```
-
-| Attribute | Type    | Required  | Description |
-| --------- | ------- | --------- | ----------- |
-| `token`   | string  | yes       | Runner's registration token |
-
-Example request:
-
-```sh
-curl --request POST "https://gitlab.example.com/ci/api/v1/runners/register" --form "token=t0k3n"
-```
-
-## Delete a Runner
-
-Used to remove a Runner.
-
-```sh
-DELETE /ci/api/v1/runners/delete
-```
-
-| Attribute | Type    | Required  | Description |
-| --------- | ------- | --------- | ----------- |
-| `token`   | string  | yes       | Unique Runner's token |
-
-Example request:
-
-```sh
-curl --request DELETE "https://gitlab.example.com/ci/api/v1/runners/delete" --form "token=t0k3n"
-```

doc/api/ci/lint.md → doc/api/lint.md

@@ -5,7 +5,7 @@
 Checks if your .gitlab-ci.yml file is valid.
 
 ```
-POST ci/lint
+POST /lint
 ```
 
 | Attribute  | Type    | Required | Description |
@@ -49,3 +49,4 @@ Example responses:
     ```
 
 [ce-5953]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5953
+

doc/ci/api/README.md

-This document was moved to a [new location](../../api/ci/README.md).

doc/ci/api/builds.md

-This document was moved to a [new location](../../api/ci/builds.md).

doc/ci/api/runners.md

-This document was moved to a [new location](../../api/ci/runners.md).

doc/topics/authentication/index.md

@@ -37,7 +37,6 @@ This page gathers all the resources for the topic **Authentication** within GitL
 - [Private Tokens](../../api/README.md#private-tokens)
 - [Impersonation tokens](../../api/README.md#impersonation-tokens)
 - [GitLab as an OAuth2 provider](../../api/oauth2.md#gitlab-as-an-oauth2-provider)
-- [GitLab Runner API - Authentication](../../api/ci/runners.md#authentication)
 
 ## Third-party resources
 

doc/user/group/index.md

@@ -55,6 +55,12 @@ By doing so:
 - John mentions everyone from his team with `@john-team`
 - John mentions only his marketing team with `@john-team/marketing`
 
+## Issues and merge requests within a group
+
+Issues and merge requests are part of projects. For a given group, view all the
+[issues](../project/issues/index.md#issues-per-group) and [merge requests](../project/merge_requests/index.md#merge-requests-per-group) across all the projects in that group,
+together in a single list view.
+
 ## Create a new group
 
 > **Notes:**

doc/user/index.md

@@ -126,6 +126,10 @@ are a tool for working faster and more effectively with your team,
 by listing all user or group mentions, as well as issues and merge
 requests you're assigned to.
 
+## Search
+
+[Search and filter](search/index.md) through groups, projects, issues, merge requests, files, code, and more.
+
 ## Snippets
 
 [Snippets](snippets.md) are code blocks that you want to store in GitLab, from which

doc/user/project/issues/index.md

@@ -7,7 +7,7 @@ of solving a problem.
 It allows you, your team, and your collaborators to share
 and discuss proposals before and while implementing them.
 
-Issues and the GitLab Issue Tracker are available in all
+GitLab Issues and the GitLab Issue Tracker are available in all
 [GitLab Products](https://about.gitlab.com/products/) as
 part of the [GitLab Workflow](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/).
 
@@ -48,11 +48,27 @@ for feature proposals and another one for bug reports.
 
 ## Issue Tracker
 
-The issue tracker is the collection of opened and closed issues created in a project.
+The Issue Tracker is the collection of opened and closed issues created in a project.
+It is available for all projects, from the moment the project is created.
 
-![Issue tracker](img/issue_tracker.png)
+Find the issue tracker by navigating to your **Project's homepage** > **Issues**.
 
-Find the issue tracker by navigating to your **Project's Dashboard** > **Issues**.
+### Issues per project
+
+When you access your project's issues, GitLab will present them in a list,
+and you can use the tabs available to quickly filter by open and closed issues.
+
+![Project issues list view](img/project_issues_list_view.png)
+
+You can also [search and filter](../../search/index.md#issues-and-merge-requests-per-project) the results more deeply with GitLab's search capacities.
+
+### Issues per group
+
+View all the issues in a group (that is, all the issues across all projects in that
+group) by navigating to **Group > Issues**. This view also has the open and closed
+issue tabs.
+
+![Group Issues list view](img/group_issues_list_view.png)
 
 ## GitLab Issues Functionalities
 
@@ -120,6 +136,12 @@ to find out more about this feature.
 With [GitLab Enterprise Edition Starter](https://about.gitlab.com/gitlab-ee/), you can also
 create various boards per project with [Multiple Issue Boards](https://docs.gitlab.com/ee/user/project/issue_board.html#multiple-issue-boards).
 
+### External Issue Tracker
+
+Alternatively to GitLab's built-in Issue Tracker, you can also use an [external
+tracker](../../../integration/external-issue-tracker.md) such as Jira, Redmine,
+or Bugzilla.
+
 ### Issue's API
 
 Read through the [API documentation](../../../api/issues.md).

doc/user/project/merge_requests/index.md

@@ -56,6 +56,23 @@ B. Consider you're a web developer writing a webpage for your company's:
 1. Once approved, your merge request is [squashed and merged](https://docs.gitlab.com/ee/user/project/merge_requests/squash_and_merge.html), and [deployed to staging with GitLab Pages](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments/) (Squash and Merge is available in GitLab Enterprise Edition Starter)
 1. Your production team [cherry picks](#cherry-pick-changes) the merge commit into production
 
+## Merge requests per project
+
+View all the merge requests within a project by navigating to **Project > Merge Requests**.
+
+When you access your project's merge requests, GitLab will present them in a list,
+and you can use the tabs available to quickly filter by open and closed. You can also [search and filter the results](../../search/index.md#issues-and-merge-requests-per-project).
+
+![Project merge requests list view](img/project_merge_requests_list_view.png)
+
+## Merge requests per group
+
+View all the merge requests in a group (that is, all the merge requests across all projects in that
+group) by navigating to **Group > Merge Requests**. This view also has the open, merged, and closed
+merge request tabs, from which you can [search and filter the results](../../search/index.md#issues-and-merge-requests-per-group).
+
+![Group Issues list view](img/group_merge_requests_list_view.png)
+
 ## Authorization for merge requests
 
 There are two main ways to have a merge request flow with GitLab:
@@ -141,7 +158,6 @@ all your changes will be available to preview by anyone with the Review Apps lin
 
 [Read more about Review Apps.](../../../ci/review_apps/index.md)
 
-
 ## Tips
 
 Here are some tips that will help you be more efficient with merge requests in
@@ -230,4 +246,4 @@ git checkout origin/merge-requests/1
 ```
 
 [protected branches]: ../protected_branches.md
-[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition"
+[ee]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition"
\ No newline at end of file

doc/user/search/index.md

@@ -27,7 +27,7 @@ on the search field on the top-right of your screen:
 
 ![shortcut to your issues and mrs](img/issues_mrs_shortcut.png)
 
-## Issues and merge requests per project
+### Issues and merge requests per project
 
 If you want to search for issues present in a specific project, navigate to
 a project's **Issues** tab, and click on the field **Search or filter results...**. It will
@@ -40,7 +40,7 @@ The same process is valid for merge requests. Navigate to your project's **Merge
 and click **Search or filter results...**. Merge requests can be filtered by author, assignee,
 milestone, and label.
 
-## Issues and merge requests per group
+### Issues and merge requests per group
 
 Similar to **Issues and merge requests per project**, you can also search for issues
 within a group. Navigate to a group's **Issues** tab and query search results in
@@ -48,6 +48,10 @@ the same way as you do for projects.
 
 ![filter issues in a group](img/group_issues_filter.png)
 
+The same process is valid for merge requests. Navigate to your project's **Merge Requests** tab.
+The search and filter UI currently uses dropdowns. In a future release, the same
+dynamic UI as above will be carried over here.
+
 ## Search history
 
 You can view recent searches by clicking on the little arrow-clock icon, which is to the left of the search input. Click the search entry to run that search again. This feature is available for issues and merge requests. Searches are stored locally in your browser.

lib/backup/repository.rb

@@ -75,7 +75,7 @@ module Backup
         path_to_project_repo = path_to_repo(project)
         path_to_project_bundle = path_to_bundle(project)
 
-        project.ensure_storage_path_exist
+        project.ensure_storage_path_exists
 
         cmd = if File.exist?(path_to_project_bundle)
                 %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo})

lib/ci/api/api.rb

-module Ci
-  module API
-    class API < Grape::API
-      include ::API::APIGuard
-      version 'v1', using: :path
-
-      rescue_from ActiveRecord::RecordNotFound do
-        rack_response({ 'message' => '404 Not found' }.to_json, 404)
-      end
-
-      # Retain 405 error rather than a 500 error for Grape 0.15.0+.
-      # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
-      rescue_from Grape::Exceptions::MethodNotAllowed do |e|
-        error! e.message, e.status, e.headers
-      end
-
-      rescue_from Grape::Exceptions::Base do |e|
-        error! e.message, e.status, e.headers
-      end
-
-      rescue_from :all do |exception|
-        handle_api_exception(exception)
-      end
-
-      content_type :txt,  'text/plain'
-      content_type :json, 'application/json'
-      format :json
-
-      helpers ::SentryHelper
-      helpers ::Ci::API::Helpers
-      helpers ::API::Helpers
-      helpers Gitlab::CurrentSettings
-
-      mount ::Ci::API::Builds
-      mount ::Ci::API::Runners
-      mount ::Ci::API::Triggers
-    end
-  end
-end

lib/ci/api/builds.rb

-module Ci
-  module API
-    # Builds API
-    class Builds < Grape::API
-      resource :builds do
-        # Runs oldest pending build by runner - Runners only
-        #
-        # Parameters:
-        #   token (required) - The uniq token of runner
-        #
-        # Example Request:
-        #   POST /builds/register
-        post "register" do
-          authenticate_runner!
-          required_attributes! [:token]
-          not_found! unless current_runner.active?
-          update_runner_info
-
-          if current_runner.is_runner_queue_value_latest?(params[:last_update])
-            header 'X-GitLab-Last-Update', params[:last_update]
-            Gitlab::Metrics.add_event(:build_not_found_cached)
-            return build_not_found!
-          end
-
-          new_update = current_runner.ensure_runner_queue_value
-
-          result = Ci::RegisterJobService.new(current_runner).execute
-
-          if result.valid?
-            if result.build
-              Gitlab::Metrics.add_event(:build_found,
-                                        project: result.build.project.full_path)
-
-              present result.build, with: Entities::BuildDetails
-            else
-              Gitlab::Metrics.add_event(:build_not_found)
-
-              header 'X-GitLab-Last-Update', new_update
-
-              build_not_found!
-            end
-          else
-            # We received build that is invalid due to concurrency conflict
-            Gitlab::Metrics.add_event(:build_invalid)
-            conflict!
-          end
-        end
-
-        # Update an existing build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   state (optional) - The state of a build
-        #   trace (optional) - The trace of a build
-        # Example Request:
-        #   PUT /builds/:id
-        put ":id" do
-          authenticate_runner!
-          build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
-          validate_build!(build)
-
-          update_runner_info
-
-          build.trace.set(params[:trace]) if params[:trace]
-
-          Gitlab::Metrics.add_event(:update_build,
-                                    project: build.project.full_path)
-
-          case params[:state].to_s
-          when 'success'
-            build.success
-          when 'failed'
-            build.drop
-          end
-        end
-
-        # Send incremental log update - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        # Body:
-        #   content of logs to append
-        # Headers:
-        #   Content-Range (required) - range of content that was sent
-        #   BUILD-TOKEN (required) - The build authorization token
-        # Example Request:
-        #   PATCH /builds/:id/trace.txt
-        patch ":id/trace.txt" do
-          build = authenticate_build!
-
-          error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
-          content_range = request.headers['Content-Range']
-          content_range = content_range.split('-')
-
-          stream_size = build.trace.append(request.body.read, content_range[0].to_i)
-          if stream_size < 0
-            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
-          end
-
-          status 202
-          header 'Build-Status', build.status
-          header 'Range', "0-#{stream_size}"
-        end
-
-        # Authorize artifacts uploading for build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        #   filesize (optional) - the size of uploaded file
-        # Example Request:
-        #   POST /builds/:id/artifacts/authorize
-        post ":id/artifacts/authorize" do
-          require_gitlab_workhorse!
-          Gitlab::Workhorse.verify_api_request!(headers)
-          not_allowed! unless Gitlab.config.artifacts.enabled
-          build = authenticate_build!
-          forbidden!('build is not running') unless build.running?
-
-          if params[:filesize]
-            file_size = params[:filesize].to_i
-            file_to_large! unless file_size < max_artifacts_size
-          end
-
-          status 200
-          content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-          Gitlab::Workhorse.artifact_upload_ok
-        end
-
-        # Upload artifacts to build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        #   file (required) - Artifacts file
-        #   expire_in (optional) - Specify when artifacts should expire (ex. 7d)
-        # Parameters (accelerated by GitLab Workhorse):
-        #   file.path - path to locally stored body (generated by Workhorse)
-        #   file.name - real filename as send in Content-Disposition
-        #   file.type - real content type as send in Content-Type
-        #   metadata.path - path to locally stored body (generated by Workhorse)
-        #   metadata.name - filename (generated by Workhorse)
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Body:
-        #   The file content
-        #
-        # Example Request:
-        #   POST /builds/:id/artifacts
-        post ":id/artifacts" do
-          require_gitlab_workhorse!
-          not_allowed! unless Gitlab.config.artifacts.enabled
-          build = authenticate_build!
-          forbidden!('Build is not running!') unless build.running?
-
-          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-          artifacts = uploaded_file(:file, artifacts_upload_path)
-          metadata = uploaded_file(:metadata, artifacts_upload_path)
-
-          bad_request!('Missing artifacts file!') unless artifacts
-          file_to_large! unless artifacts.size < max_artifacts_size
-
-          build.artifacts_file = artifacts
-          build.artifacts_metadata = metadata
-          build.artifacts_expire_in =
-            params['expire_in'] ||
-            Gitlab::CurrentSettings.current_application_settings
-              .default_artifacts_expire_in
-
-          if build.save
-            present(build, with: Entities::BuildDetails)
-          else
-            render_validation_error!(build)
-          end
-        end
-
-        # Download the artifacts file from build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Example Request:
-        #   GET /builds/:id/artifacts
-        get ":id/artifacts" do
-          build = authenticate_build!
-          artifacts_file = build.artifacts_file
-
-          unless artifacts_file.exists?
-            not_found!
-          end
-
-          unless artifacts_file.file_storage?
-            return redirect_to build.artifacts_file.url
-          end
-
-          present_file!(artifacts_file.path, artifacts_file.filename)
-        end
-
-        # Remove the artifacts file from build - Runners only
-        #
-        # Parameters:
-        #   id (required) - The ID of a build
-        #   token (required) - The build authorization token
-        # Headers:
-        #   BUILD-TOKEN (required) - The build authorization token, the same as token
-        # Example Request:
-        #   DELETE /builds/:id/artifacts
-        delete ":id/artifacts" do
-          build = authenticate_build!
-
-          status(200)
-          build.erase_artifacts!
-        end
-      end
-    end
-  end
-end

lib/ci/api/entities.rb

-module Ci
-  module API
-    module Entities
-      class Commit < Grape::Entity
-        expose :id, :sha, :project_id, :created_at
-        expose :status, :finished_at, :duration
-        expose :git_commit_message, :git_author_name, :git_author_email
-      end
-
-      class CommitWithBuilds < Commit
-        expose :builds
-      end
-
-      class ArtifactFile < Grape::Entity
-        expose :filename, :size
-      end
-
-      class BuildOptions < Grape::Entity
-        expose :image
-        expose :services
-        expose :artifacts
-        expose :cache
-        expose :dependencies
-        expose :after_script
-      end
-
-      class Build < Grape::Entity
-        expose :id, :ref, :tag, :sha, :status
-        expose :name, :token, :stage
-        expose :project_id
-        expose :project_name
-        expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
-      end
-
-      class BuildCredentials < Grape::Entity
-        expose :type, :url, :username, :password
-      end
-
-      class BuildDetails < Build
-        expose :commands
-        expose :repo_url
-        expose :before_sha
-        expose :allow_git_fetch
-        expose :token
-        expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
-
-        expose :options do |model|
-          # This part ensures that output of old API is still the same after adding support
-          # for extended docker configuration options, used by new API
-          #
-          # I'm leaving this here, not in the model, because it should be removed at the same time
-          # when old API will be removed (planned for August 2017).
-          model.options.dup.tap do |options|
-            options[:image] = options[:image][:name] if options[:image].is_a?(Hash)
-            options[:services]&.map! do |service|
-              if service.is_a?(Hash)
-                service[:name]
-              else
-                service
-              end
-            end
-          end
-        end
-
-        expose :timeout do |model|
-          model.timeout
-        end
-
-        expose :variables
-        expose :depends_on_builds, using: Build
-
-        expose :credentials, using: BuildCredentials
-      end
-
-      class Runner < Grape::Entity
-        expose :id, :token
-      end
-
-      class RunnerProject < Grape::Entity
-        expose :id, :project_id, :runner_id
-      end
-
-      class WebHook < Grape::Entity
-        expose :id, :project_id, :url
-      end
-
-      class TriggerRequest < Grape::Entity
-        expose :id, :variables
-        expose :pipeline, using: Commit, as: :commit
-      end
-    end
-  end
-end

lib/ci/api/helpers.rb

-module Ci
-  module API
-    module Helpers
-      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze
-      BUILD_TOKEN_PARAM = :token
-      UPDATE_RUNNER_EVERY = 10 * 60
-
-      def authenticate_runners!
-        forbidden! unless runner_registration_token_valid?
-      end
-
-      def authenticate_runner!
-        forbidden! unless current_runner
-      end
-
-      def authenticate_build!
-        build = Ci::Build.find_by_id(params[:id])
-
-        validate_build!(build) do
-          forbidden! unless build_token_valid?(build)
-        end
-
-        build
-      end
-
-      def validate_build!(build)
-        not_found! unless build
-
-        yield if block_given?
-
-        project = build.project
-        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
-        forbidden!('Build has been erased!') if build.erased?
-      end
-
-      def runner_registration_token_valid?
-        ActiveSupport::SecurityUtils.variable_size_secure_compare(
-          params[:token],
-          current_application_settings.runners_registration_token)
-      end
-
-      def build_token_valid?(build)
-        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
-
-        # We require to also check `runners_token` to maintain compatibility with old version of runners
-        token && (build.valid_token?(token) || build.project.valid_runners_token?(token))
-      end
-
-      def update_runner_info
-        return unless update_runner?
-
-        current_runner.contacted_at = Time.now
-        current_runner.assign_attributes(get_runner_version_from_params)
-        current_runner.save if current_runner.changed?
-      end
-
-      def update_runner?
-        # Use a random threshold to prevent beating DB updates.
-        # It generates a distribution between [40m, 80m].
-        #
-        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
-
-        current_runner.contacted_at.nil? ||
-          (Time.now - current_runner.contacted_at) >= contacted_at_max_age
-      end
-
-      def build_not_found!
-        if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
-          no_content!
-        else
-          not_found!
-        end
-      end
-
-      def current_runner
-        @runner ||= Runner.find_by_token(params[:token].to_s)
-      end
-
-      def get_runner_version_from_params
-        return unless params["info"].present?
-        attributes_for_keys(%w(name version revision platform architecture), params["info"])
-      end
-
-      def max_artifacts_size
-        current_application_settings.max_artifacts_size.megabytes.to_i
-      end
-    end
-  end
-end

lib/ci/api/runners.rb

-module Ci
-  module API
-    class Runners < Grape::API
-      resource :runners do
-        desc 'Delete a runner'
-        params do
-          requires :token, type: String, desc: 'The unique token of the runner'
-        end
-        delete "delete" do
-          authenticate_runner!
-
-          status(200)
-          Ci::Runner.find_by_token(params[:token]).destroy
-        end
-
-        desc 'Register a new runner' do
-          success Entities::Runner
-        end
-        params do
-          requires :token, type: String, desc: 'The unique token of the runner'
-          optional :description, type: String, desc: 'The description of the runner'
-          optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for'
-          optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs'
-          optional :locked, type: Boolean, desc: 'Lock this runner for this specific project'
-        end
-        post "register" do
-          runner_params = declared(params, include_missing: false).except(:token)
-
-          runner =
-            if runner_registration_token_valid?
-              # Create shared runner. Requires admin access
-              Ci::Runner.create(runner_params.merge(is_shared: true))
-            elsif project = Project.find_by(runners_token: params[:token])
-              # Create a specific runner for project.
-              project.runners.create(runner_params)
-            end
-
-          return forbidden! unless runner
-
-          if runner.id
-            runner.update(get_runner_version_from_params)
-            present runner, with: Entities::Runner
-          else
-            not_found!
-          end
-        end
-      end
-    end
-  end
-end

lib/ci/api/triggers.rb

-module Ci
-  module API
-    class Triggers < Grape::API
-      resource :projects do
-        desc 'Trigger a GitLab CI project build' do
-          success Entities::TriggerRequest
-        end
-        params do
-          requires :id, type: Integer, desc: 'The ID of a CI project'
-          requires :ref, type: String, desc: "The name of project's branch or tag"
-          requires :token, type: String, desc: 'The unique token of the trigger'
-          optional :variables, type: Hash, desc: 'Optional build variables'
-        end
-        post ":id/refs/:ref/trigger" do
-          project = Project.find_by(ci_id: params[:id])
-          trigger = Ci::Trigger.find_by_token(params[:token])
-          not_found! unless project && trigger
-          unauthorized! unless trigger.project == project
-
-          # Validate variables
-          variables = params[:variables].to_h
-          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
-            render_api_error!('variables needs to be a map of key-valued strings', 400)
-          end
-
-          # create request and trigger builds
-          result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables)
-          pipeline = result.pipeline
-
-          if pipeline.persisted?
-            present result.trigger_request, with: Entities::TriggerRequest
-          else
-            render_validation_error!(pipeline)
-          end
-        end
-      end
-    end
-  end
-end

lib/gitlab/background_migration/migrate_stage_status.rb

+module Gitlab
+  module BackgroundMigration
+    class MigrateStageStatus
+      STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+                   failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+      class Build < ActiveRecord::Base
+        self.table_name = 'ci_builds'
+
+        scope :latest, -> { where(retried: [false, nil]) }
+        scope :created, -> { where(status: 'created') }
+        scope :running, -> { where(status: 'running') }
+        scope :pending, -> { where(status: 'pending') }
+        scope :success, -> { where(status: 'success') }
+        scope :failed, -> { where(status: 'failed')  }
+        scope :canceled, -> { where(status: 'canceled')  }
+        scope :skipped, -> { where(status: 'skipped')  }
+        scope :manual, -> { where(status: 'manual')  }
+
+        scope :failed_but_allowed, -> do
+          where(allow_failure: true, status: [:failed, :canceled])
+        end
+
+        scope :exclude_ignored, -> do
+          where("allow_failure = ? OR status IN (?)",
+            false, %w[created pending running success skipped])
+        end
+
+        def self.status_sql
+          scope_relevant = latest.exclude_ignored
+          scope_warnings = latest.failed_but_allowed
+
+          builds = scope_relevant.select('count(*)').to_sql
+          created = scope_relevant.created.select('count(*)').to_sql
+          success = scope_relevant.success.select('count(*)').to_sql
+          manual = scope_relevant.manual.select('count(*)').to_sql
+          pending = scope_relevant.pending.select('count(*)').to_sql
+          running = scope_relevant.running.select('count(*)').to_sql
+          skipped = scope_relevant.skipped.select('count(*)').to_sql
+          canceled = scope_relevant.canceled.select('count(*)').to_sql
+          warnings = scope_warnings.select('count(*) > 0').to_sql
+
+          <<-SQL.strip_heredoc
+            (CASE
+              WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]}
+              WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]}
+              WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]}
+              WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]}
+              WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]}
+              WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]}
+              WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]}
+              WHEN (#{created}) > 0 THEN #{STATUSES[:running]}
+              ELSE #{STATUSES[:failed]}
+            END)
+          SQL
+        end
+      end
+
+      def perform(start_id, stop_id)
+        status_sql = Build
+          .where('ci_builds.commit_id = ci_stages.pipeline_id')
+          .where('ci_builds.stage = ci_stages.name')
+          .status_sql
+
+        sql = <<-SQL
+          UPDATE ci_stages SET status = (#{status_sql})
+            WHERE ci_stages.status IS NULL
+            AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+        SQL
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+  end
+end

lib/gitlab/git/repository.rb

@@ -64,7 +64,6 @@ module Gitlab
       end
 
       delegate  :empty?,
-                :bare?,
                 to: :rugged
 
       delegate :exists?, to: :gitaly_repository_client
@@ -126,6 +125,8 @@ module Gitlab
       # This is to work around a bug in libgit2 that causes in-memory refs to
       # be stale/invalid when packed-refs is changed.
       # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474
       def find_branch(name, force_reload = false)
         reload_rugged if force_reload
 
@@ -231,10 +232,6 @@ module Gitlab
         branch_names + tag_names
       end
 
-      def has_commits?
-        !empty?
-      end
-
       # Discovers the default branch based on the repository's available branches
       #
       # - If no branches are present, returns nil
@@ -574,6 +571,8 @@ module Gitlab
       end
 
       # Delete the specified branch from the repository
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def delete_branch(branch_name)
         rugged.branches.delete(branch_name)
       end
@@ -583,6 +582,8 @@ module Gitlab
       # Examples:
       #   create_branch("feature")
       #   create_branch("other-feature", "master")
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def create_branch(ref, start_point = "HEAD")
         rugged_ref = rugged.branches.create(ref, start_point)
         target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
@@ -592,38 +593,26 @@ module Gitlab
         raise InvalidRef.new("Invalid reference #{start_point}")
       end
 
-      # Return an array of this repository's remote names
-      def remote_names
-        rugged.remotes.each_name.to_a
-      end
-
       # Delete the specified remote from this repository.
       def remote_delete(remote_name)
         rugged.remotes.delete(remote_name)
+        nil
       end
 
-      # Add a new remote to this repository.  Returns a Rugged::Remote object
+      # Add a new remote to this repository.
       def remote_add(remote_name, url)
         rugged.remotes.create(remote_name, url)
+        nil
       end
 
       # Update the specified remote using the values in the +options+ hash
       #
       # Example
       # repo.update_remote("origin", url: "path/to/repo")
-      def remote_update(remote_name, options = {})
+      def remote_update(remote_name, url:)
         # TODO: Implement other remote options
-        rugged.remotes.set_url(remote_name, options[:url]) if options[:url]
-      end
-
-      # Fetch the specified remote
-      def fetch(remote_name)
-        rugged.remotes[remote_name].fetch
-      end
-
-      # Push +*refspecs+ to the remote identified by +remote_name+.
-      def push(remote_name, *refspecs)
-        rugged.remotes[remote_name].push(refspecs)
+        rugged.remotes.set_url(remote_name, url)
+        nil
       end
 
       AUTOCRLF_VALUES = {

lib/gitlab/gitaly_client/commit_service.rb

@@ -80,8 +80,8 @@ module Gitlab
       def tree_entries(repository, revision, path)
         request = Gitaly::GetTreeEntriesRequest.new(
           repository: @gitaly_repo,
-          revision: revision,
-          path: path.presence || '.'
+          revision: GitalyClient.encode(revision),
+          path: path.present? ? GitalyClient.encode(path) : '.'
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request)

lib/gitlab/import_export/import_export.yml

@@ -98,6 +98,7 @@ excluded_attributes:
     - :last_activity_at
     - :last_repository_updated_at
     - :last_repository_check_at
+    - :storage_version
   snippets:
     - :expired_at
   merge_request_diff:

lib/gitlab/job_waiter.rb

 module Gitlab
   # JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
+  #
+  # Its use requires the cooperation of the sidekiq jobs themselves. Set up the
+  # waiter, then start the jobs, passing them its `key`. Their `perform` methods
+  # should look like:
+  #
+  #     def perform(args, notify_key)
+  #       # do work
+  #     ensure
+  #       ::Gitlab::JobWaiter.notify(notify_key, jid)
+  #     end
+  #
+  # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs
+  # push to that array when done. Once the waiter has popped `count` items, it
+  # knows all the jobs are done.
   class JobWaiter
-    # The sleep interval between checking keys, in seconds.
-    INTERVAL = 0.1
+    def self.notify(key, jid)
+      Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) }
+    end
+
+    attr_reader :key, :jobs_remaining, :finished
 
-    # jobs - The job IDs to wait for.
-    def initialize(jobs)
-      @jobs = jobs
+    # jobs_remaining - the number of jobs left to wait for
+    def initialize(jobs_remaining)
+      @key = "gitlab:job_waiter:#{SecureRandom.uuid}"
+      @jobs_remaining = jobs_remaining
+      @finished = []
     end
 
     # Waits for all the jobs to be completed.
@@ -15,13 +34,33 @@ module Gitlab
     #           ensures we don't indefinitely block a caller in case a job takes
     #           long to process, or is never processed.
     def wait(timeout = 10)
-      start = Time.current
+      deadline = Time.now.utc + timeout
+
+      Gitlab::Redis::SharedState.with do |redis|
+        # Fallback key expiry: allow a long grace period to reduce the chance of
+        # a job pushing to an expired key and recreating it
+        redis.expire(key, [timeout * 2, 10.minutes.to_i].max)
+
+        while jobs_remaining > 0
+          # Redis will not take fractional seconds. Prefer waiting too long over
+          # not waiting long enough
+          seconds_left = (deadline - Time.now.utc).ceil
 
-      while (Time.current - start) <= timeout
-        break if SidekiqStatus.all_completed?(@jobs)
+          # Redis interprets 0 as "wait forever", so skip the final `blpop` call
+          break if seconds_left <= 0
 
-        sleep(INTERVAL) # to not overload Redis too much.
+          list, jid = redis.blpop(key, timeout: seconds_left)
+          break unless list && jid # timed out
+
+          @finished << jid
+          @jobs_remaining -= 1
+        end
+
+        # All jobs have finished, so expire the key immediately
+        redis.expire(key, 0) if jobs_remaining == 0
       end
+
+      finished
     end
   end
 end

lib/gitlab/sidekiq_throttler.rb

@@ -3,6 +3,8 @@ module Gitlab
     class << self
       def execute!
         if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
+          require 'sidekiq-limit_fetch'
+
           Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
             Sidekiq::Queue[queue].limit = queue_limit
           end

lib/tasks/gitlab/import.rake

@@ -11,6 +11,12 @@ namespace :gitlab do
     #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
+      if Project.current_application_settings.hashed_storage_enabled
+        puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
+
+        exit 1
+      end
+
       Gitlab.config.repositories.storages.each_value do |repository_storage|
         git_base_path = repository_storage['path']
         repos_to_import = Dir.glob(git_base_path + '/**/*.git')

spec/factories/ci/stages.rb

@@ -15,4 +15,12 @@ FactoryGirl.define do
                                     warnings: warnings)
     end
   end
+
+  factory :ci_stage_entity, class: Ci::Stage do
+    project factory: :project
+    pipeline factory: :ci_empty_pipeline
+
+    name 'test'
+    status 'pending'
+  end
 end

spec/factories/projects.rb

@@ -81,6 +81,10 @@ FactoryGirl.define do
       archived true
     end
 
+    trait :hashed do
+      storage_version Project::LATEST_STORAGE_VERSION
+    end
+
     trait :access_requestable do
       request_access_enabled true
     end

spec/features/atom/users_spec.rb

@@ -41,6 +41,8 @@ describe "User Feed"  do
                target_project: project,
                description: "Here is the fix: ![an image](image.png)")
       end
+      let(:push_event) { create(:push_event, project: project, author: user) }
+      let!(:push_event_payload) { create(:push_event_payload, event: push_event) }
 
       before do
         project.team << [user, :master]
@@ -70,6 +72,10 @@ describe "User Feed"  do
       it 'has XHTML summaries in merge request descriptions' do
         expect(body).to match /Here is the fix: <a[^>]*><img[^>]*\/><\/a>/
       end
+
+      it 'has push event commit ID' do
+        expect(body).to have_content(Commit.truncate_sha(push_event.commit_id))
+      end
     end
   end
 

spec/features/groups_spec.rb

@@ -104,18 +104,15 @@ feature 'Group' do
     end
 
     context 'as group owner' do
-      let(:user) { create(:user) }
+      it 'creates a nested group' do
+        user = create(:user)
 
-      before do
         group.add_owner(user)
         sign_out(:user)
         sign_in(user)
 
         visit subgroups_group_path(group)
         click_link 'New Subgroup'
-      end
-
-      it 'creates a nested group' do
         fill_in 'Group path', with: 'bar'
         click_button 'Create group'
 
@@ -123,6 +120,16 @@ feature 'Group' do
         expect(page).to have_content("Group 'bar' was successfully created.")
       end
     end
+
+    context 'when nested group feature is disabled' do
+      it 'renders 404' do
+        allow(Group).to receive(:supports_nested_groups?).and_return(false)
+
+        visit subgroups_group_path(group)
+
+        expect(page.status_code).to eq(404)
+      end
+    end
   end
 
   it 'checks permissions to avoid exposing groups by parent_id' do

spec/helpers/events_helper_spec.rb

@@ -62,6 +62,12 @@ describe EventsHelper do
       expect(helper.event_note(input)).to eq(expected)
     end
 
+    it 'preserves data-src for lazy images' do
+      input = "![ImageTest](/uploads/test.png)"
+      image_url = "data-src=\"/uploads/test.png\""
+      expect(helper.event_note(input)).to match(image_url)
+    end
+
     context 'labels formatting' do
       let(:input) { 'this should be ~label_1' }
 

spec/javascripts/fixtures/project_select_combo_button.html.haml

 .project-item-select-holder
   %input.project-item-select{ data: { group_id: '12345' , relative_path: 'issues/new' } }
-  %a.new-project-item-link{ data: { label: 'New issue' }, href: ''}
+  %a.new-project-item-link{ data: { label: 'New issue', type: 'issues' }, href: ''}
     %i.fa.fa-spinner.spin
   %a.new-project-item-select-button
     %i.fa.fa-caret-down

spec/javascripts/project_select_combo_button_spec.js

@@ -101,5 +101,40 @@ describe('Project Select Combo Button', function () {
       window.localStorage.clear();
     });
   });
+
+  describe('deriveTextVariants', function () {
+    beforeEach(function () {
+      this.mockExecutionContext = {
+        resourceType: '',
+        resourceLabel: '',
+      };
+
+      this.comboButton = new ProjectSelectComboButton(this.projectSelectInput);
+
+      this.method = this.comboButton.deriveTextVariants.bind(this.mockExecutionContext);
+    });
+
+    it('correctly derives test variants for merge requests', function () {
+      this.mockExecutionContext.resourceType = 'merge_requests';
+      this.mockExecutionContext.resourceLabel = 'New merge request';
+
+      const returnedVariants = this.method();
+
+      expect(returnedVariants.localStorageItemType).toBe('new-merge-request');
+      expect(returnedVariants.defaultTextPrefix).toBe('New merge request');
+      expect(returnedVariants.presetTextSuffix).toBe('merge request');
+    });
+
+    it('correctly derives text variants for issues', function () {
+      this.mockExecutionContext.resourceType = 'issues';
+      this.mockExecutionContext.resourceLabel = 'New issue';
+
+      const returnedVariants = this.method();
+
+      expect(returnedVariants.localStorageItemType).toBe('new-issue');
+      expect(returnedVariants.defaultTextPrefix).toBe('New issue');
+      expect(returnedVariants.presetTextSuffix).toBe('issue');
+    });
+  });
 });
 

spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateStageStatus, :migration, schema: 20170711145320 do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+               failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+  before do
+    projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1')
+    pipelines.create!(id: 1, project_id: 1, ref: 'master', sha: 'adf43c3a')
+    stages.create!(id: 1, pipeline_id: 1, project_id: 1, name: 'test', status: nil)
+    stages.create!(id: 2, pipeline_id: 1, project_id: 1, name: 'deploy', status: nil)
+  end
+
+  context 'when stage status is known' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'success')
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'running')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:running]
+      expect(stages.second.status).to eq STATUSES[:failed]
+    end
+  end
+
+  context 'when stage status is not known' do
+    it 'sets a skipped stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:skipped]
+      expect(stages.second.status).to eq STATUSES[:skipped]
+    end
+  end
+
+  context 'when stage status includes status of a retried job' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'canceled')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed', retried: true)
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:canceled]
+      expect(stages.second.status).to eq STATUSES[:success]
+    end
+  end
+
+  context 'when some job in the stage is blocked / manual' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'failed')
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'manual')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success', when: 'manual')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:manual]
+      expect(stages.second.status).to eq STATUSES[:success]
+    end
+  end
+
+  def create_job(project:, pipeline:, stage:, status:, **opts)
+    stages = { test: 1, build: 2, deploy: 3 }
+
+    jobs.create!(project_id: project, commit_id: pipeline,
+                 stage_idx: stages[stage.to_sym], stage: stage,
+                 status: status, **opts)
+  end
+end

spec/lib/gitlab/git/repository_spec.rb

@@ -235,18 +235,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     it { is_expected.to be < 2 }
   end
 
-  describe '#has_commits?' do
-    it { expect(repository.has_commits?).to be_truthy }
-  end
-
   describe '#empty?' do
     it { expect(repository.empty?).to be_falsey }
   end
 
-  describe '#bare?' do
-    it { expect(repository.bare?).to be_truthy }
-  end
-
   describe '#ref_names' do
     let(:ref_names) { repository.ref_names }
     subject { ref_names }
@@ -441,15 +433,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_names" do
-    let(:remotes) { repository.remote_names }
-
-    it "should have one entry: 'origin'" do
-      expect(remotes.size).to eq(1)
-      expect(remotes.first).to eq("origin")
-    end
-  end
-
   describe "#refs_hash" do
     let(:refs) { repository.refs_hash }
 

spec/lib/gitlab/gitaly_client/commit_service_spec.rb

@@ -111,6 +111,20 @@ describe Gitlab::GitalyClient::CommitService do
 
       client.tree_entries(repository, revision, path)
     end
+
+    context 'with UTF-8 params strings' do
+      let(:revision) { "branch\u011F" }
+      let(:path) { "foo/\u011F.txt" }
+
+      it 'handles string encodings correctly' do
+        expect_any_instance_of(Gitaly::CommitService::Stub)
+          .to receive(:get_tree_entries)
+          .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
+          .and_return([])
+
+        client.tree_entries(repository, revision, path)
+      end
+    end
   end
 
   describe '#find_commit' do

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -227,6 +227,8 @@ Ci::Pipeline:
 Ci::Stage:
 - id
 - name
+- status
+- lock_version
 - project_id
 - pipeline_id
 - created_at

spec/lib/gitlab/job_waiter_spec.rb

 require 'spec_helper'
 
 describe Gitlab::JobWaiter do
-  describe '#wait' do
-    let(:waiter) { described_class.new(%w(a)) }
-    it 'returns when all jobs have been completed' do
-      expect(Gitlab::SidekiqStatus).to receive(:all_completed?).with(%w(a))
-        .and_return(true)
+  describe '.notify' do
+    it 'pushes the jid to the named queue' do
+      key = 'gitlab:job_waiter:foo'
+      jid = 1
 
-      expect(waiter).not_to receive(:sleep)
+      redis = double('redis')
+      expect(Gitlab::Redis::SharedState).to receive(:with).and_yield(redis)
+      expect(redis).to receive(:lpush).with(key, jid)
 
-      waiter.wait
+      described_class.notify(key, jid)
     end
+  end
+
+  describe '#wait' do
+    let(:waiter) { described_class.new(2) }
 
-    it 'sleeps between checking the job statuses' do
-      expect(Gitlab::SidekiqStatus).to receive(:all_completed?)
-        .with(%w(a))
-        .and_return(false, true)
+    it 'returns when all jobs have been completed' do
+      described_class.notify(waiter.key, 'a')
+      described_class.notify(waiter.key, 'b')
 
-      expect(waiter).to receive(:sleep).with(described_class::INTERVAL)
+      result = nil
+      expect { Timeout.timeout(1) { result = waiter.wait(2) } }.not_to raise_error
 
-      waiter.wait
+      expect(result).to contain_exactly('a', 'b')
     end
 
-    it 'returns when timing out' do
-      expect(waiter).not_to receive(:sleep)
-      waiter.wait(0)
+    it 'times out if not all jobs complete' do
+      described_class.notify(waiter.key, 'a')
+
+      result = nil
+      expect { Timeout.timeout(2) { result = waiter.wait(1) } }.not_to raise_error
+
+      expect(result).to contain_exactly('a')
     end
   end
 end

spec/lib/gitlab/sidekiq_throttler_spec.rb

 require 'spec_helper'
 
 describe Gitlab::SidekiqThrottler do
-  before do
-    Sidekiq.options[:concurrency] = 35
-
-    stub_application_setting(
-      sidekiq_throttling_enabled: true,
-      sidekiq_throttling_factor: 0.1,
-      sidekiq_throttling_queues: %w[build project_cache]
-    )
-  end
-
   describe '#execute!' do
-    it 'sets limits on the selected queues' do
-      described_class.execute!
+    context 'when job throttling is enabled' do
+      before do
+        Sidekiq.options[:concurrency] = 35
+
+        stub_application_setting(
+          sidekiq_throttling_enabled: true,
+          sidekiq_throttling_factor: 0.1,
+          sidekiq_throttling_queues: %w[build project_cache]
+        )
+      end
+
+      it 'requires sidekiq-limit_fetch' do
+        expect(described_class).to receive(:require).with('sidekiq-limit_fetch').and_call_original
+
+        described_class.execute!
+      end
+
+      it 'sets limits on the selected queues' do
+        described_class.execute!
+
+        expect(Sidekiq::Queue['build'].limit).to eq 4
+        expect(Sidekiq::Queue['project_cache'].limit).to eq 4
+      end
+
+      it 'does not set limits on other queues' do
+        described_class.execute!
 
-      expect(Sidekiq::Queue['build'].limit).to eq 4
-      expect(Sidekiq::Queue['project_cache'].limit).to eq 4
+        expect(Sidekiq::Queue['merge'].limit).to be_nil
+      end
     end
 
-    it 'does not set limits on other queues' do
-      described_class.execute!
+    context 'when job throttling is disabled' do
+      it 'does not require sidekiq-limit_fetch' do
+        expect(described_class).not_to receive(:require).with('sidekiq-limit_fetch')
 
-      expect(Sidekiq::Queue['merge'].limit).to be_nil
+        described_class.execute!
+      end
     end
   end
 end

spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb

@@ -4,7 +4,7 @@ require Rails.root.join('db', 'post_migrate', '20170502101023_cleanup_namespacel
 describe CleanupNamespacelessPendingDeleteProjects do
   before do
     # Stub after_save callbacks that will fail when Project has no namespace
-    allow_any_instance_of(Project).to receive(:ensure_storage_path_exist).and_return(nil)
+    allow_any_instance_of(Project).to receive(:ensure_storage_path_exists).and_return(nil)
     allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
   end
 

spec/migrations/migrate_stage_id_reference_in_background_spec.rb

@@ -2,19 +2,6 @@ require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20170628080858_migrate_stage_id_reference_in_background')
 
 describe MigrateStageIdReferenceInBackground, :migration, :sidekiq do
-  matcher :be_scheduled_migration do |delay, *expected|
-    match do |migration|
-      BackgroundMigrationWorker.jobs.any? do |job|
-        job['args'] == [migration, expected] &&
-          job['at'].to_i == (delay.to_i + Time.now.to_i)
-      end
-    end
-
-    failure_message do |migration|
-      "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!"
-    end
-  end
-
   let(:jobs) { table(:ci_builds) }
   let(:stages) { table(:ci_stages) }
   let(:pipelines) { table(:ci_pipelines) }