Merge branch 'master-ce' into artifact-format-v2

CHANGELOG.md

@@ -2,6 +2,24 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 11.1.3 (2018-07-27)
+
+### Fixed (8 changes, 1 of them is from the community)
+
+- Rework some projects table indexes around repository_storage field. !20377
+- Fix navigation to First and Next discussion on MR Changes tab. !20434
+- Fix showing outdated discussions on Changes tab. !20445
+- Fix autosave and ESC confirmation issues for MR discussions. !20569
+- Fix rendering of the context lines in MR diffs page. !20642
+- Don't overflow project/group dropdown results. !20704 (gfyoung)
+- Fixed IDE not opening JSON files. !20798
+- Disable Gitaly timeouts when creating or restoring backups. !20810
+
+### Performance (1 change)
+
+- Reduces the client side memory footprint on merge requests. !20744
+
+
 ## 11.1.2 (2018-07-26)
 
 ### Security (4 changes)

GITLAB_SHELL_VERSION

-7.1.5
+7.2.0

Gemfile.rails5.lock

@@ -364,7 +364,7 @@ GEM
     grape-entity (0.7.1)
       activesupport (>= 4.0)
       multi_json (>= 1.3.2)
-    grape-path-helpers (1.0.5)
+    grape-path-helpers (1.0.6)
       activesupport (>= 4, < 5.1)
       grape (~> 1.0)
       rake (~> 12)
@@ -378,7 +378,8 @@ GEM
       google-protobuf (~> 3.1)
       googleapis-common-protos-types (~> 1.0.0)
       googleauth (>= 0.5.1, < 0.7)
-    haml (4.0.7)
+    haml (5.0.4)
+      temple (>= 0.8.0)
       tilt
     haml_lint (0.26.0)
       haml (>= 4.0, < 5.1)
@@ -400,7 +401,7 @@ GEM
     hipchat (1.5.2)
       httparty
       mimemagic
-    html-pipeline (2.8.3)
+    html-pipeline (2.8.4)
       activesupport (>= 2)
       nokogiri (>= 1.4)
     html2text (0.2.0)
@@ -518,7 +519,7 @@ GEM
     net-ssh (5.0.1)
     netrc (0.11.0)
     nio4r (2.3.1)
-    nokogiri (1.8.3)
+    nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
     nokogumbo (1.5.0)
       nokogiri
@@ -821,7 +822,7 @@ GEM
       et-orbi (~> 1.0)
     rugged (0.27.2)
     safe_yaml (1.0.4)
-    sanitize (4.6.5)
+    sanitize (4.6.6)
       crass (~> 1.0.2)
       nokogiri (>= 1.4.4)
       nokogumbo (~> 1.4)
@@ -915,7 +916,7 @@ GEM
       rack (>= 1, < 3)
     thor (0.19.4)
     thread_safe (0.3.6)
-    tilt (2.0.6)
+    tilt (2.0.8)
     timecop (0.8.1)
     timfel-krb5-auth (0.8.3)
     toml (0.1.2)

app/assets/javascripts/badges/components/badge.vue

@@ -93,7 +93,7 @@ export default {
         <icon
           :size="16"
           class="prepend-left-8 append-right-8"
-          name="doc_image"
+          name="doc-image"
           aria-hidden="true"
         />
       </div>

app/assets/javascripts/ide/components/commit_sidebar/stage_button.vue

@@ -56,7 +56,7 @@ export default {
     >
       <icon
         :size="12"
-        name="more"
+        name="ellipsis_h"
       />
     </button>
     <div class="dropdown-menu dropdown-menu-right">

app/assets/javascripts/pipelines/components/graph/dropdown_job_component.vue

@@ -14,7 +14,7 @@ import tooltip from '../../../vue_shared/directives/tooltip';
  *   "id": 4256,
  *   "name": "test",
  *   "status": {
- *     "icon": "icon_status_success",
+ *     "icon": "status_success",
  *     "text": "passed",
  *     "label": "passed",
  *     "group": "success",

app/assets/javascripts/pipelines/components/graph/job_component.vue

@@ -13,7 +13,7 @@ import tooltip from '../../../vue_shared/directives/tooltip';
  *   "id": 4256,
  *   "name": "test",
  *   "status": {
- *     "icon": "icon_status_success",
+ *     "icon": "status_success",
  *     "text": "passed",
  *     "label": "passed",
  *     "group": "success",

app/assets/stylesheets/snippets.scss

@@ -22,8 +22,8 @@
     height: 16px;
     background-size: cover;
 
-    &.gl-snippet-icon-doc_code { background-position: 0 0;  }
-    &.gl-snippet-icon-doc_text {  background-position: 0 -16px; }
+    &.gl-snippet-icon-doc-code { background-position: 0 0;  }
+    &.gl-snippet-icon-doc-text {  background-position: 0 -16px; }
     &.gl-snippet-icon-download {  background-position: 0 -32px; }
   }
 

app/finders/admin/projects_finder.rb

@@ -7,7 +7,7 @@ class Admin::ProjectsFinder
   end
 
   def execute
-    items = Project.without_deleted.with_statistics
+    items = Project.without_deleted.with_statistics.with_route
     items = by_namespace_id(items)
     items = by_visibilty_level(items)
     items = by_with_push(items)
@@ -16,7 +16,7 @@ class Admin::ProjectsFinder
     items = by_archived(items)
     items = by_personal(items)
     items = by_name(items)
-    items = items.includes(namespace: [:owner])
+    items = items.includes(namespace: [:owner, :route])
     sort(items).page(params[:page])
   end
 

app/helpers/snippets_helper.rb

@@ -116,7 +116,7 @@ module SnippetsHelper
                         raw_project_snippet_url(@snippet.project, @snippet)
                       end
 
-    link_to external_snippet_icon('doc_code'), snippet_raw_url, class: 'btn', target: '_blank', rel: 'noopener noreferrer', title: 'Open raw'
+    link_to external_snippet_icon('doc-code'), snippet_raw_url, class: 'btn', target: '_blank', rel: 'noopener noreferrer', title: 'Open raw'
   end
 
   def embedded_snippet_download_button

app/mailers/notify.rb

@@ -124,7 +124,7 @@ class Notify < BaseMailer
 
       fallback_reply_message_id = "<reply-#{reply_key}@#{Gitlab.config.gitlab.host}>".freeze
       headers['References'] ||= []
-      headers['References'] << fallback_reply_message_id
+      headers['References'].unshift(fallback_reply_message_id)
 
       @reply_by_email = true
     end
@@ -158,7 +158,7 @@ class Notify < BaseMailer
   def mail_answer_thread(model, headers = {})
     headers['Message-ID'] = "<#{SecureRandom.hex}@#{Gitlab.config.gitlab.host}>"
     headers['In-Reply-To'] = message_id(model)
-    headers['References'] = message_id(model)
+    headers['References'] = [message_id(model)]
 
     headers[:subject]&.prepend('Re: ')
 

app/models/concerns/routable.rb

@@ -90,34 +90,17 @@ module Routable
   end
 
   def full_name
-    if route && route.name.present?
-      @full_name ||= route.name # rubocop:disable Gitlab/ModuleWithInstanceVariables
-    else
-      update_route if persisted?
-
-      build_full_name
-    end
+    route&.name || build_full_name
   end
 
-  # Every time `project.namespace.becomes(Namespace)` is called for polymorphic_path,
-  # a new instance is instantiated, and we end up duplicating the same query to retrieve
-  # the route. Caching this per request ensures that even if we have multiple instances,
-  # we will not have to duplicate work, avoiding N+1 queries in some cases.
   def full_path
-    return uncached_full_path unless RequestStore.active? && persisted?
-
-    RequestStore[full_path_key] ||= uncached_full_path
+    route&.path || build_full_path
   end
 
   def full_path_components
     full_path.split('/')
   end
 
-  def expires_full_path_cache
-    RequestStore.delete(full_path_key) if RequestStore.active?
-    @full_path = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
-  end
-
   def build_full_path
     if parent && path
       parent.full_path + '/' + path
@@ -138,16 +121,6 @@ module Routable
     self.errors[:path].concat(route_path_errors) if route_path_errors
   end
 
-  def uncached_full_path
-    if route && route.path.present?
-      @full_path ||= route.path # rubocop:disable Gitlab/ModuleWithInstanceVariables
-    else
-      update_route if persisted?
-
-      build_full_path
-    end
-  end
-
   def full_name_changed?
     name_changed? || parent_changed?
   end
@@ -156,10 +129,6 @@ module Routable
     path_changed? || parent_changed?
   end
 
-  def full_path_key
-    @full_path_key ||= "routable/full_path/#{self.class.name}/#{self.id}"
-  end
-
   def build_full_name
     if parent && name
       parent.human_name + ' / ' + name
@@ -168,18 +137,9 @@ module Routable
     end
   end
 
-  def update_route
-    return if Gitlab::Database.read_only?
-
-    prepare_route
-    route.save
-  end
-
   def prepare_route
     route || build_route(source: self)
     route.path = build_full_path
     route.name = build_full_name
-    @full_path = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
-    @full_name = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
   end
 end

app/models/concerns/storage/legacy_namespace.rb

@@ -11,8 +11,6 @@ module Storage
                      Namespace.find(parent_id_was) # raise NotFound early if needed
                    end
 
-      expires_full_path_cache
-
       move_repositories
 
       if parent_changed?

app/models/deploy_token.rb

 class DeployToken < ActiveRecord::Base
   include Expirable
   include TokenAuthenticatable
+  include PolicyActor
   add_authentication_token_field :token
 
   AVAILABLE_SCOPES = %i(read_repository read_registry).freeze
@@ -58,10 +59,6 @@ class DeployToken < ActiveRecord::Base
     write_attribute(:expires_at, value.presence || Forever.date)
   end
 
-  def admin?
-    false
-  end
-
   private
 
   def ensure_at_least_one_scope

app/models/namespace.rb

@@ -304,7 +304,6 @@ class Namespace < ActiveRecord::Base
 
   def write_projects_repository_config
     all_projects.find_each do |project|
-      project.expires_full_path_cache # we need to clear cache to validate renames correctly
       project.write_repository_config
     end
   end

app/models/project.rb

@@ -1238,8 +1238,6 @@ class Project < ActiveRecord::Base
     return true if skip_disk_validation
     return false unless repository_storage
 
-    expires_full_path_cache # we need to clear cache to validate renames correctly
-
     # Check if repository with same path already exists on disk we can
     # skip this for the hashed storage because the path does not change
     if legacy_storage? && repository_with_same_path_already_exists?
@@ -1618,7 +1616,6 @@ class Project < ActiveRecord::Base
     # When we import a project overwriting the original project, there
     # is a move operation. In that case we don't want to send the instructions.
     send_move_instructions(full_path_was) unless import_started?
-    expires_full_path_cache
 
     self.old_path_with_namespace = full_path_was
     SystemHooksService.new.execute_hooks_for(self, :rename)

app/policies/concerns/policy_actor.rb

+# frozen_string_literal: true
+
+# Include this module if we want to pass something else than the user to
+# check policies. This defines several methods which the policy checker
+# would call and check.
+module PolicyActor
+  extend ActiveSupport::Concern
+
+  def blocked?
+    false
+  end
+
+  def admin?
+    false
+  end
+
+  def external?
+    false
+  end
+
+  def internal?
+    false
+  end
+
+  def access_locked?
+    false
+  end
+
+  def required_terms_not_accepted?
+    false
+  end
+
+  def can_create_group
+    false
+  end
+end

app/serializers/pipeline_serializer.rb

@@ -11,10 +11,15 @@ class PipelineSerializer < BaseSerializer
         :retryable_builds,
         :cancelable_statuses,
         :trigger_requests,
-        :project,
         :manual_actions,
         :artifacts,
-        { pending_builds: :project }
+        {
+          pending_builds: :project,
+          project: [:route, { namespace: :route }],
+          artifacts: {
+            project: [:route, { namespace: :route }]
+          }
+        }
       ])
     end
 

app/services/projects/transfer_service.rb

@@ -77,7 +77,6 @@ module Projects
         Gitlab::PagesTransfer.new.move_project(project.path, @old_namespace.full_path, @new_namespace.full_path)
 
         project.old_path_with_namespace = @old_path
-        project.expires_full_path_cache
 
         write_repository_config(@new_path)
 

app/views/admin/projects/_projects.html.haml

@@ -17,7 +17,7 @@
             - if project.archived
               %span.badge.badge-warning archived
           .title
-            = link_to [:admin, project.namespace.becomes(Namespace), project] do
+            = link_to(admin_namespace_project_path(project.namespace, project)) do
               .dash-project-avatar
                 .avatar-container.s40
                   = project_icon(project, alt: '', class: 'avatar project-avatar s40')

app/views/layouts/header/_default.html.haml

@@ -67,5 +67,5 @@
 
       %button.navbar-toggler.d-block.d-sm-none{ type: 'button' }
         %span.sr-only= _("Toggle navigation")
-        = sprite_icon('more', size: 12, css_class: 'more-icon js-navbar-toggle-right')
+        = sprite_icon('ellipsis_h', size: 12, css_class: 'more-icon js-navbar-toggle-right')
         = sprite_icon('close', size: 12, css_class: 'close-icon js-navbar-toggle-left')

app/views/layouts/nav/sidebar/_group.html.haml

@@ -15,7 +15,7 @@
         = nav_link(path: ['groups#show', 'groups#activity', 'groups#subgroups', 'analytics#show'], html_options: { class: 'home' }) do
           = link_to group_path(@group) do
             .nav-icon-container
-              = sprite_icon('project')
+              = sprite_icon('home')
             %span.nav-item-name
               = _('Overview')
 

app/views/layouts/nav/sidebar/_profile.html.haml

@@ -110,7 +110,7 @@
       = nav_link(controller: :gpg_keys) do
         = link_to profile_gpg_keys_path do
           .nav-icon-container
-            = sprite_icon('key-2')
+            = sprite_icon('key-modern')
           %span.nav-item-name
             = _('GPG Keys')
         %ul.sidebar-sub-level-items.is-fly-out-only

app/views/layouts/nav/sidebar/_project.html.haml

@@ -11,7 +11,7 @@
       = nav_link(path: sidebar_projects_paths, html_options: { class: 'home' }) do
         = link_to project_path(@project), class: 'shortcuts-project' do
           .nav-icon-container
-            = sprite_icon('project')
+            = sprite_icon('home')
           %span.nav-item-name
             = _('Project')
 
@@ -40,7 +40,7 @@
         = nav_link(controller: sidebar_repository_paths) do
           = link_to project_tree_path(@project), class: 'shortcuts-tree' do
             .nav-icon-container
-              = sprite_icon('doc_text')
+              = sprite_icon('doc-text')
             %span.nav-item-name
               = _('Repository')
 

app/views/projects/jobs/_sidebar.html.haml

@@ -86,7 +86,7 @@
       - HasStatus::ORDERED_STATUSES.each do |build_status|
         - builds.select{|build| build.status == build_status}.each do |build|
           .build-job{ class: sidebar_build_class(build, @build), data: { stage: build.stage } }
-            - tooltip = sanitize(build.tooltip_message)
+            - tooltip = sanitize(build.tooltip_message.dup)
             = link_to(project_job_path(@project, build), data: { toggle: 'tooltip', html: 'true', title: tooltip, container: 'body' }) do
               = sprite_icon('arrow-right', size:16, css_class: 'icon-arrow-right')
               %span{ class: "ci-status-icon-#{build.status}" }

app/views/shared/snippets/_embed.html.haml

@@ -2,7 +2,7 @@
 .gitlab-embed-snippets
   .js-file-title.file-title-flex-parent
     .file-header-content
-      = external_snippet_icon('doc_text')
+      = external_snippet_icon('doc-text')
 
       %strong.file-title-name
         %a.gitlab-embedded-snippets-title{ href: url_for(only_path: false, overwrite_params: nil) }

changelogs/unreleased/4525-fix-project-indexes.yml

----
-title: Rework some projects table indexes around repository_storage field
-merge_request: 20377
-author:
-type: fixed

changelogs/unreleased/48817-fix-mr-changes-discussion-navigation.yml

----
-title: Fix navigation to First and Next discussion on MR Changes tab
-merge_request: 20434
-author:
-type: fixed

changelogs/unreleased/_acet-fix-expanding-context-lines.yml

----
-title: Fix rendering of the context lines in MR diffs page
-merge_request: 20642
-author:
-type: fixed

changelogs/unreleased/_acet-fix-mr-autosave.yml

----
-title: Fix autosave and ESC confirmation issues for MR discussions
-merge_request: 20569
-author:
-type: fixed

changelogs/unreleased/_acet-fix-outdated-discussions.yml

----
-title: Fix showing outdated discussions on Changes tab
-merge_request: 20445
-author:
-type: fixed

changelogs/unreleased/ide-edit-json-files.yml

----
-title: Fixed IDE not opening JSON files
-merge_request: 20798
-author:
-type: fixed

changelogs/unreleased/project-dropdown-list-overflow.yml

----
-title: Don't overflow project/group dropdown results
-merge_request: 20704
-author: gfyoung
-type: fixed

changelogs/unreleased/rails5-update-gemfile-lock-2.yml

+---
+title: Rails5 update Gemfile.rails5.lock
+merge_request: 20858
+author: Jasper Maes
+type: fixed

changelogs/unreleased/stop-dynamic-routable-creation.yml

+---
+title: Stop dynamically creating project and namespace routes
+merge_request: 20313
+author:
+type: performance

changelogs/unreleased/tc-reorder-mail-notify-references.yml

+---
+title: Put fallback reply-key address first in the References header
+merge_request: 20871
+author:
+type: changed

changelogs/unreleased/tz-mr-refactor-memory-reduction.yml

----
-title: Reduces the client side memory footprint on merge requests
-merge_request: 20744
-author:
-type: performance

changelogs/unreleased/zj-backup-timeout.yml

----
-title: Disable Gitaly timeouts when creating or restoring backups
-merge_request: 20810
-author:
-type: fixed

config/application.rb

@@ -43,6 +43,7 @@ module Gitlab
                                      #{config.root}/app/models/members
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
+                                     #{config.root}/app/policies/concerns
                                      #{config.root}/app/services/concerns
                                      #{config.root}/app/serializers/concerns
                                      #{config.root}/app/finders/concerns

db/migrate/20180702124358_remove_orphaned_routes.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveOrphanedRoutes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+    include EachBatch
+
+    def self.orphaned_namespace_routes
+      where(source_type: 'Namespace')
+        .where('NOT EXISTS ( SELECT 1 FROM namespaces WHERE namespaces.id = routes.source_id )')
+    end
+
+    def self.orphaned_project_routes
+      where(source_type: 'Project')
+        .where('NOT EXISTS ( SELECT 1 FROM projects WHERE projects.id = routes.source_id )')
+    end
+  end
+
+  def up
+    # Some of these queries can take up to 10 seconds to run on GitLab.com,
+    # which is pretty close to our 15 second statement timeout. To ensure a
+    # smooth deployment procedure we disable the statement timeouts for this
+    # migration, just in case.
+    disable_statement_timeout
+
+    # On GitLab.com there are around 4000 orphaned project routes, and around
+    # 150 orphaned namespace routes.
+    [
+      Route.orphaned_project_routes,
+      Route.orphaned_namespace_routes
+    ].each do |relation|
+      relation.each_batch(of: 1_000) do |batch|
+        batch.delete_all
+      end
+    end
+  end
+
+  def down
+    # There is no way to restore orphaned routes, and this doesn't make any
+    # sense anyway.
+  end
+end

db/migrate/20180702134423_generate_missing_routes.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+# This migration generates missing routes for any projects and namespaces that
+# don't already have a route.
+#
+# On GitLab.com this would insert 611 project routes, and 0 namespace routes.
+# The exact number could vary per instance, so we take care of both just in
+# case.
+class GenerateMissingRoutes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+  end
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+  end
+
+  module Routable
+    def build_full_path
+      if parent && path
+        parent.build_full_path + '/' + path
+      else
+        path
+      end
+    end
+
+    def build_full_name
+      if parent && name
+        parent.human_name + ' / ' + name
+      else
+        name
+      end
+    end
+
+    def human_name
+      build_full_name
+    end
+
+    def attributes_for_insert
+      time = Time.zone.now
+
+      {
+        # We can't use "self.class.name" here as that would include the
+        # migration namespace.
+        source_type: source_type_for_route,
+        source_id: id,
+        created_at: time,
+        updated_at: time,
+        name: build_full_name,
+
+        # The route path might already be taken. Instead of trying to generate a
+        # new unique name on every conflict, we just append the row ID to the
+        # route path.
+        path: "#{build_full_path}-#{id}"
+      }
+    end
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include EachBatch
+    include GenerateMissingRoutes::Routable
+
+    belongs_to :namespace, class_name: 'GenerateMissingRoutes::Namespace'
+
+    has_one :route,
+      as: :source,
+      inverse_of: :source,
+      class_name: 'GenerateMissingRoutes::Route'
+
+    alias_method :parent, :namespace
+    alias_attribute :parent_id, :namespace_id
+
+    def self.without_routes
+      where(
+        'NOT EXISTS (
+          SELECT 1
+          FROM routes
+          WHERE source_type = ?
+          AND source_id = projects.id
+        )',
+        'Project'
+      )
+    end
+
+    def source_type_for_route
+      'Project'
+    end
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    include EachBatch
+    include GenerateMissingRoutes::Routable
+
+    belongs_to :parent, class_name: 'GenerateMissingRoutes::Namespace'
+    belongs_to :owner, class_name: 'GenerateMissingRoutes::User'
+
+    has_one :route,
+      as: :source,
+      inverse_of: :source,
+      class_name: 'GenerateMissingRoutes::Route'
+
+    def self.without_routes
+      where(
+        'NOT EXISTS (
+          SELECT 1
+          FROM routes
+          WHERE source_type = ?
+          AND source_id = namespaces.id
+        )',
+        'Namespace'
+      )
+    end
+
+    def source_type_for_route
+      'Namespace'
+    end
+  end
+
+  def up
+    [Namespace, Project].each do |model|
+      model.without_routes.each_batch(of: 100) do |batch|
+        rows = batch.map(&:attributes_for_insert)
+
+        Gitlab::Database.bulk_insert(:routes, rows)
+      end
+    end
+  end
+
+  def down
+    # Removing routes we previously generated makes no sense.
+  end
+end

lib/gitlab/git/conflict/resolver.rb

@@ -17,7 +17,7 @@ module Gitlab
           end
         rescue GRPC::FailedPrecondition => e
           raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message)
-        rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e
+        rescue GRPC::BadStatus => e
           raise Gitlab::Git::CommandError.new(e)
         end
 

lib/gitlab/git/repository.rb

@@ -558,10 +558,12 @@ module Gitlab
           if is_enabled
             gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev)
           else
+            Gitlab::GitalyClient::StorageSettings.allow_disk_access do
               OperationService.new(user, self).update_branch(branch_name, newrev, oldrev)
             end
           end
         end
+      end
 
       def rm_branch(branch_name, user:)
         wrapped_gitaly_errors do
@@ -832,18 +834,9 @@ module Gitlab
         Gitlab::Git.check_namespace!(source_repository)
         source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository)
 
-        message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled|
-          if is_enabled
-            gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref)
-          else
-            # When removing this code, also remove source_repository#path
-            # to remove deprecated method calls
-            local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref)
-          end
-        end
-
-        # Make sure ref was created, and raise Rugged::ReferenceError when not
-        raise Rugged::ReferenceError, message if status != 0
+        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
+        message, status = run_git(args, env: source_repository.fetch_env)
+        raise Gitlab::Git::CommandError, message if status != 0
 
         target_ref
       end
@@ -1244,17 +1237,6 @@ module Gitlab
         gitaly_repository_client.apply_gitattributes(revision)
       end
 
-      def local_fetch_ref(source_path, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
-        run_git(args)
-      end
-
-      def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
-
-        run_git(args, env: source_repository.fetch_env)
-      end
-
       def gitaly_delete_refs(*ref_names)
         gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any?
       end

lib/gitlab/git/repository_mirroring.rb

@@ -6,10 +6,12 @@ module Gitlab
           if is_enabled
             gitaly_ref_client.remote_branches(remote_name)
           else
+            Gitlab::GitalyClient::StorageSettings.allow_disk_access do
               rugged_remote_branches(remote_name)
             end
           end
         end
+      end
 
       private
 

lib/gitlab/import_export/merge_request_parser.rb

@@ -27,7 +27,11 @@ module Gitlab
 
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1295
       def fetch_ref
-        @project.repository.fetch_ref(@project.repository, source_ref: @diff_head_sha, target_ref: @merge_request.source_branch)
+        target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch
+
+        unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref)
+          Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}")
+        end
       end
 
       def branch_exists?(branch_name)

package.json

@@ -18,7 +18,7 @@
     "webpack-prod": "NODE_ENV=production webpack --config config/webpack.config.js"
   },
   "dependencies": {
-    "@gitlab-org/gitlab-svgs": "^1.26.0",
+    "@gitlab-org/gitlab-svgs": "^1.27.0",
     "@gitlab-org/gitlab-ui": "1.0.5",
     "autosize": "^4.0.0",
     "axios": "^0.17.1",

spec/controllers/projects/pipelines_controller_spec.rb

@@ -55,10 +55,8 @@ describe Projects::PipelinesController do
         stub_feature_flags(ci_pipeline_persisted_stages: false)
       end
 
-      it 'returns JSON with serialized pipelines', :request_store do
-        queries = ActiveRecord::QueryRecorder.new do
+      it 'returns JSON with serialized pipelines' do
         get_pipelines_index_json
-        end
 
         expect(response).to have_gitlab_http_status(:ok)
         expect(response).to match_response_schema('pipeline')
@@ -73,8 +71,14 @@ describe Projects::PipelinesController do
         json_response.dig('pipelines', 0, 'details', 'stages').tap do |stages|
           expect(stages.count).to eq 3
         end
+      end
+
+      it 'does not execute N+1 queries' do
+        queries = ActiveRecord::QueryRecorder.new do
+          get_pipelines_index_json
+        end
 
-        expect(queries.count).to be_within(5).of(30)
+        expect(queries.count).to be <= 36
       end
     end
 

spec/fixtures/emails/commands_in_reply.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/fixtures/emails/commands_only_reply.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/fixtures/emails/reply_without_subaddressing_and_key_inside_references.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/fixtures/emails/reply_without_subaddressing_and_key_inside_references_with_a_comma.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>,<exchange@microsoft.com>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>,<exchange@microsoft.com>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/fixtures/emails/update_commands_only_reply.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/fixtures/emails/valid_reply.eml

@@ -8,7 +8,7 @@ From: Jake the Dog <jake@adventuretime.ooo>
 To: reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo
 Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
 In-Reply-To: <issue_1@localhost>
-References: <issue_1@localhost> <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost>
+References: <reply-59d8df8370b7e95c5a49fbf86aeb2c93@localhost> <issue_1@localhost>
 Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
 Mime-Version: 1.0
 Content-Type: text/plain;

spec/helpers/snippets_helper_spec.rb

@@ -7,13 +7,13 @@ describe SnippetsHelper do
     it 'gives view raw button of embedded snippets for project snippets' do
       @snippet = create(:project_snippet, :public)
 
-      expect(embedded_snippet_raw_button.to_s).to eq("<a class=\"btn\" target=\"_blank\" rel=\"noopener noreferrer\" title=\"Open raw\" href=\"#{raw_project_snippet_url(@snippet.project, @snippet)}\">#{external_snippet_icon('doc_code')}</a>")
+      expect(embedded_snippet_raw_button.to_s).to eq("<a class=\"btn\" target=\"_blank\" rel=\"noopener noreferrer\" title=\"Open raw\" href=\"#{raw_project_snippet_url(@snippet.project, @snippet)}\">#{external_snippet_icon('doc-code')}</a>")
     end
 
     it 'gives view raw button of embedded snippets for personal snippets' do
       @snippet = create(:personal_snippet, :public)
 
-      expect(embedded_snippet_raw_button.to_s).to eq("<a class=\"btn\" target=\"_blank\" rel=\"noopener noreferrer\" title=\"Open raw\" href=\"#{raw_snippet_url(@snippet)}\">#{external_snippet_icon('doc_code')}</a>")
+      expect(embedded_snippet_raw_button.to_s).to eq("<a class=\"btn\" target=\"_blank\" rel=\"noopener noreferrer\" title=\"Open raw\" href=\"#{raw_snippet_url(@snippet)}\">#{external_snippet_icon('doc-code')}</a>")
     end
   end
 

spec/javascripts/pipelines/graph/dropdown_job_component_spec.js

@@ -12,7 +12,7 @@ describe('dropdown job component', () => {
         id: 4256,
         name: '<img src=x onerror=alert(document.domain)>',
         status: {
-          icon: 'icon_status_success',
+          icon: 'status_success',
           text: 'passed',
           label: 'passed',
           tooltip: 'passed',
@@ -31,7 +31,7 @@ describe('dropdown job component', () => {
         id: 4299,
         name: 'test',
         status: {
-          icon: 'icon_status_success',
+          icon: 'status_success',
           text: 'passed',
           label: 'passed',
           tooltip: 'passed',
@@ -50,7 +50,7 @@ describe('dropdown job component', () => {
     name: 'rspec:linux',
     size: 2,
     status: {
-      icon: 'icon_status_success',
+      icon: 'status_success',
       text: 'passed',
       label: 'passed',
       tooltip: 'passed',

spec/javascripts/pipelines/graph/job_component_spec.js

@@ -169,7 +169,7 @@ describe('pipeline graph job component', () => {
           id: 4259,
           name: '<img src=x onerror=alert(document.domain)>',
           status: {
-            icon: 'icon_status_success',
+            icon: 'status_success',
             label: 'success',
             tooltip: 'failed',
           },

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -16,7 +16,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         @shared = @project.import_export_shared
         allow(@shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/')
 
-        allow_any_instance_of(Repository).to receive(:fetch_ref).and_return(true)
+        allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true)
         allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
 
         expect_any_instance_of(Gitlab::Git::Repository).to receive(:create_branch).with('feature', 'DCBA')

spec/migrations/generate_missing_routes_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180702134423_generate_missing_routes.rb')
+
+describe GenerateMissingRoutes, :migration do
+  describe '#up' do
+    let(:namespaces) { table(:namespaces) }
+    let(:projects) { table(:projects) }
+    let(:routes) { table(:routes) }
+
+    it 'creates routes for projects without a route' do
+      namespace = namespaces.create!(name: 'GitLab', path: 'gitlab')
+
+      routes.create!(
+        path: 'gitlab',
+        source_type: 'Namespace',
+        source_id: namespace.id
+      )
+
+      project = projects.create!(
+        name: 'GitLab CE',
+        path: 'gitlab-ce',
+        namespace_id: namespace.id
+      )
+
+      described_class.new.up
+
+      route = routes.where(source_type: 'Project').take
+
+      expect(route.source_id).to eq(project.id)
+      expect(route.path).to eq("gitlab/gitlab-ce-#{project.id}")
+    end
+
+    it 'creates routes for namespaces without a route' do
+      namespace = namespaces.create!(name: 'GitLab', path: 'gitlab')
+
+      described_class.new.up
+
+      route = routes.where(source_type: 'Namespace').take
+
+      expect(route.source_id).to eq(namespace.id)
+      expect(route.path).to eq("gitlab-#{namespace.id}")
+    end
+
+    it 'does not create routes for namespaces that already have a route' do
+      namespace = namespaces.create!(name: 'GitLab', path: 'gitlab')
+
+      routes.create!(
+        path: 'gitlab',
+        source_type: 'Namespace',
+        source_id: namespace.id
+      )
+
+      described_class.new.up
+
+      expect(routes.count).to eq(1)
+    end
+
+    it 'does not create routes for projects that already have a route' do
+      namespace = namespaces.create!(name: 'GitLab', path: 'gitlab')
+
+      routes.create!(
+        path: 'gitlab',
+        source_type: 'Namespace',
+        source_id: namespace.id
+      )
+
+      project = projects.create!(
+        name: 'GitLab CE',
+        path: 'gitlab-ce',
+        namespace_id: namespace.id
+      )
+
+      routes.create!(
+        path: 'gitlab/gitlab-ce',
+        source_type: 'Project',
+        source_id: project.id
+      )
+
+      described_class.new.up
+
+      expect(routes.count).to eq(2)
+    end
+  end
+end

spec/models/concerns/routable_spec.rb

@@ -12,16 +12,6 @@ describe Group, 'Routable' do
     it { is_expected.to have_many(:redirect_routes).dependent(:destroy) }
   end
 
-  describe 'GitLab read-only instance' do
-    it 'does not save route if route is not present' do
-      group.route.path = ''
-      allow(Gitlab::Database).to receive(:read_only?).and_return(true)
-      expect(group).to receive(:update_route).and_call_original
-
-      expect { group.full_path }.to change { Route.count }.by(0)
-    end
-  end
-
   describe 'Callbacks' do
     it 'creates route record on create' do
       expect(group.route.path).to eq(group.path)
@@ -131,29 +121,6 @@ describe Group, 'Routable' do
 
     it { expect(group.full_path).to eq(group.path) }
     it { expect(nested_group.full_path).to eq("#{group.full_path}/#{nested_group.path}") }
-
-    context 'with RequestStore active', :request_store do
-      it 'does not load the route table more than once' do
-        group.expires_full_path_cache
-        expect(group).to receive(:uncached_full_path).once.and_call_original
-
-        3.times { group.full_path }
-        expect(group.full_path).to eq(group.path)
-      end
-    end
-  end
-
-  describe '#expires_full_path_cache' do
-    context 'with RequestStore active', :request_store do
-      it 'expires the full_path cache' do
-        expect(group.full_path).to eq('foo')
-
-        group.route.update(path: 'bar', name: 'bar')
-        group.expires_full_path_cache
-
-        expect(group.full_path).to eq('bar')
-      end
-    end
   end
 
   describe '#full_name' do

spec/models/namespace_spec.rb

@@ -323,6 +323,16 @@ describe Namespace do
 
       parent.update(path: 'mygroup_new')
 
+      # Routes are loaded when creating the projects, so we need to manually
+      # reload them for the below code to be aware of the above UPDATE.
+      [
+        project_in_parent_group,
+        hashed_project_in_subgroup,
+        legacy_project_in_subgroup
+      ].each do |project|
+        project.route.reload
+      end
+
       expect(project_rugged(project_in_parent_group).config['gitlab.fullpath']).to eq "mygroup_new/#{project_in_parent_group.path}"
       expect(project_rugged(hashed_project_in_subgroup).config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{hashed_project_in_subgroup.path}"
       expect(project_rugged(legacy_project_in_subgroup).config['gitlab.fullpath']).to eq "mygroup_new/mysubgroup/#{legacy_project_in_subgroup.path}"

spec/models/project_spec.rb

@@ -2958,8 +2958,6 @@ describe Project do
 
         expect(project).to receive(:expire_caches_before_rename)
 
-        expect(project).to receive(:expires_full_path_cache)
-
         project.rename_repo
       end
 
@@ -3119,8 +3117,6 @@ describe Project do
 
         expect(project).to receive(:expire_caches_before_rename)
 
-        expect(project).to receive(:expires_full_path_cache)
-
         project.rename_repo
       end
 

spec/models/repository_spec.rb

@@ -1129,16 +1129,12 @@ describe Repository do
       end
 
       it 'raises Rugged::ReferenceError' do
-        raise_reference_error = raise_error(Rugged::ReferenceError) do |err|
-          expect(err.cause).to be_nil
-        end
-
         expect do
           Gitlab::Git::OperationService.new(git_user, target_project.repository.raw_repository)
             .with_branch('feature',
                          start_repository: project.repository.raw_repository,
                          &:itself)
-        end.to raise_reference_error
+        end.to raise_error(Gitlab::Git::CommandError)
       end
     end
 

spec/policies/concerns/policy_actor_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PolicyActor do
+  it 'implements all the methods from user' do
+    methods = subject.instance_methods
+
+    # User.instance_methods do not return all methods until an instance is
+    # initialized. So here we just use an instance
+    expect(build(:user).methods).to include(*methods)
+  end
+end

spec/serializers/pipeline_serializer_spec.rb

@@ -125,7 +125,7 @@ describe PipelineSerializer do
         it 'verifies number of queries', :request_store do
           recorded = ActiveRecord::QueryRecorder.new { subject }
 
-          expect(recorded.count).to be_within(2).of(27)
+          expect(recorded.count).to be_within(2).of(31)
           expect(recorded.cached_count).to eq(0)
         end
       end
@@ -144,7 +144,7 @@ describe PipelineSerializer do
           # pipeline. With the same ref this check is cached but if refs are
           # different then there is an extra query per ref
           # https://gitlab.com/gitlab-org/gitlab-ce/issues/46368
-          expect(recorded.count).to be_within(2).of(30)
+          expect(recorded.count).to be_within(2).of(34)
           expect(recorded.cached_count).to eq(0)
         end
       end

spec/services/projects/transfer_service_spec.rb

@@ -31,12 +31,6 @@ describe Projects::TransferService do
       transfer_project(project, user, group)
     end
 
-    it 'expires full_path cache' do
-      expect(project).to receive(:expires_full_path_cache)
-
-      transfer_project(project, user, group)
-    end
-
     it 'invalidates the user\'s personal_project_count cache' do
       expect(user).to receive(:invalidate_personal_projects_count)
 

spec/support/shared_examples/notify_shared_examples.rb

@@ -77,7 +77,7 @@ shared_examples 'a thread answer email with reply-by-email enabled' do
     aggregate_failures do
       is_expected.to have_header('Message-ID', /\A<.*@#{host}>\Z/)
       is_expected.to have_header('In-Reply-To', "<#{route_key}@#{host}>")
-      is_expected.to have_header('References',  /\A<#{route_key}@#{host}> <reply\-.*@#{host}>\Z/ )
+      is_expected.to have_header('References', /\A<reply\-.*@#{host}> <#{route_key}@#{host}>\Z/ )
       is_expected.to have_subject(/^Re: /)
     end
   end

yarn.lock

@@ -78,9 +78,9 @@
     lodash "^4.2.0"
     to-fast-properties "^2.0.0"
 
-"@gitlab-org/gitlab-svgs@^1.23.0", "@gitlab-org/gitlab-svgs@^1.26.0":
-  version "1.26.0"
-  resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.26.0.tgz#d89c633e866d031a9e4787b05eacc7144c3a7791"
+"@gitlab-org/gitlab-svgs@^1.23.0", "@gitlab-org/gitlab-svgs@^1.27.0":
+  version "1.27.0"
+  resolved "https://registry.yarnpkg.com/@gitlab-org/gitlab-svgs/-/gitlab-svgs-1.27.0.tgz#638e70399ebd59e503732177316bb9a18bf7a13f"
 
 "@gitlab-org/gitlab-ui@1.0.5":
   version "1.0.5"