Merge branch 'ce-to-ee-2017-08-24' into 'master'

CE upstream: Thursday

Closes gitaly#488 and gitlab-ce#36622

See merge request !2752

Gemfile

@@ -412,7 +412,7 @@ gem 'sshkey', '~> 1.9.0'
 # Required for ED25519 SSH host key support
 group :ed25519 do
   gem 'rbnacl-libsodium'
-  gem 'rbnacl', '~> 3.2'
+  gem 'rbnacl', '~> 4.0'
   gem 'bcrypt_pbkdf', '~> 1.0'
 end
 

Gemfile.lock

@@ -227,7 +227,7 @@ GEM
       multi_json
     fast_gettext (1.4.0)
     ffaker (2.4.0)
-    ffi (1.9.10)
+    ffi (1.9.18)
     flay (2.8.1)
       erubis (~> 2.7.0)
       path_expander (~> 1.0)
@@ -711,7 +711,7 @@ GEM
     rake (12.0.0)
     rblineprof (0.3.6)
       debugger-ruby_core_source (~> 1.3)
-    rbnacl (3.4.0)
+    rbnacl (4.0.2)
       ffi
     rbnacl-libsodium (1.0.11)
       rbnacl (>= 3.0.1)
@@ -1145,7 +1145,7 @@ DEPENDENCIES
   rainbow (~> 2.2)
   raindrops (~> 0.18)
   rblineprof (~> 0.3.6)
-  rbnacl (~> 3.2)
+  rbnacl (~> 4.0)
   rbnacl-libsodium
   rdoc (~> 4.2)
   re2 (~> 1.1.1)

app/assets/stylesheets/pages/commits.scss

@@ -290,6 +290,9 @@
 
 
 .gpg-status-box {
+  padding: 2px 10px;
+  margin-right: $gl-padding;
+
   &:empty {
     display: none;
   }
@@ -318,7 +321,6 @@
   &.valid {
     svg {
       border: 1px solid $brand-success;
-
       fill: $brand-success;
     }
   }
@@ -326,7 +328,6 @@
   &.invalid {
     svg {
       border: 1px solid $common-gray-light;
-
       fill: $common-gray-light;
     }
   }

app/controllers/admin/projects_controller.rb

@@ -3,9 +3,9 @@ class Admin::ProjectsController < Admin::ApplicationController
   before_action :group, only: [:show, :transfer]
 
   def index
-    finder    = Admin::ProjectsFinder.new(params: params, current_user: current_user)
-    @projects = finder.execute
-    @sort     = finder.sort
+    params[:sort] ||= 'latest_activity_desc'
+    @sort = params[:sort]
+    @projects = Admin::ProjectsFinder.new(params: params, current_user: current_user).execute
 
     respond_to do |format|
       format.html

app/finders/admin/projects_finder.rb

 class Admin::ProjectsFinder
-  attr_reader :sort, :namespace_id, :visibility_level, :with_push,
-              :abandoned, :last_repository_check_failed, :archived,
-              :personal, :name, :page, :current_user
+  attr_reader :params, :current_user
 
   def initialize(params:, current_user:)
+    @params = params
     @current_user = current_user
-    @sort = params.fetch(:sort) { 'latest_activity_desc' }
-    @namespace_id = params[:namespace_id]
-    @visibility_level = params[:visibility_level]
-    @with_push = params[:with_push]
-    @abandoned = params[:abandoned]
-    @last_repository_check_failed = params[:last_repository_check_failed]
-    @archived = params[:archived]
-    @personal = params[:personal]
-    @name = params[:name]
-    @page = params[:page]
   end
 
   def execute
     items = Project.without_deleted.with_statistics
-    items = items.in_namespace(namespace_id) if namespace_id.present?
-    items = items.where(visibility_level: visibility_level) if visibility_level.present?
-    items = items.with_push if with_push.present?
-    items = items.abandoned if abandoned.present?
-    items = items.where(last_repository_check_failed: true) if last_repository_check_failed.present?
-    items = items.non_archived unless archived.present?
-    items = items.personal(current_user) if personal.present?
-    items = items.search(name) if name.present?
-    items = items.sort(sort)
-    items.includes(:namespace).order("namespaces.path, projects.name ASC").page(page)
+    items = by_namespace_id(items)
+    items = by_visibilty_level(items)
+    items = by_with_push(items)
+    items = by_abandoned(items)
+    items = by_last_repository_check_failed(items)
+    items = by_archived(items)
+    items = by_personal(items)
+    items = by_name(items)
+    items = sort(items)
+    items.includes(:namespace).order("namespaces.path, projects.name ASC").page(params[:page])
+  end
+
+  private
+
+  def by_namespace_id(items)
+    params[:namespace_id].present? ? items.in_namespace(params[:namespace_id]) : items
+  end
+
+  def by_visibilty_level(items)
+    params[:visibility_level].present? ? items.where(visibility_level: params[:visibility_level]) : items
+  end
+
+  def by_with_push(items)
+    params[:with_push].present? ? items.with_push : items
+  end
+
+  def by_abandoned(items)
+    params[:abandoned].present? ? items.abandoned : items
+  end
+
+  def by_last_repository_check_failed(items)
+    params[:last_repository_check_failed].present? ? items.where(last_repository_check_failed: true) : items
+  end
+
+  def by_archived(items)
+    if params[:archived] == 'only'
+      items.archived
+    elsif params[:archived].blank?
+      items.non_archived
+    else
+      items
+    end
+  end
+
+  def by_personal(items)
+    params[:personal].present? ? items.personal(current_user) : items
+  end
+
+  def by_name(items)
+    params[:name].present? ? items.search(params[:name]) : items
+  end
+
+  def sort(items)
+    sort = params.fetch(:sort) { 'latest_activity_desc' }
+    items.sort(sort)
   end
 end

app/finders/projects_finder.rb

@@ -125,9 +125,18 @@ class ProjectsFinder < UnionFinder
   end
 
   def by_archived(projects)
-    # Back-compatibility with the places where `params[:archived]` can be set explicitly to `false`
-    params[:non_archived] = !Gitlab::Utils.to_boolean(params[:archived]) if params.key?(:archived)
-
-    params[:non_archived] ? projects.non_archived : projects
+    if params[:non_archived]
+      projects.non_archived
+    elsif params.key?(:archived) # Back-compatibility with the places where `params[:archived]` can be set explicitly to `false`
+      if params[:archived] == 'only'
+        projects.archived
+      elsif Gitlab::Utils.to_boolean(params[:archived])
+        projects
+      else
+        projects.non_archived
+      end
+    else
+      projects
+    end
   end
 end

app/helpers/commits_helper.rb

@@ -114,7 +114,7 @@ module CommitsHelper
   end
 
   def commit_signature_badge_classes(additional_classes)
-    %w(btn status-box gpg-status-box) + Array(additional_classes)
+    %w(btn gpg-status-box) + Array(additional_classes)
   end
 
   protected

app/models/ci/pipeline.rb

@@ -406,7 +406,8 @@ module Ci
     def predefined_variables
       [
         { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
-        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true }
+        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
+        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
       ]
     end
 

app/models/issue.rb

@@ -61,7 +61,10 @@ class Issue < ActiveRecord::Base
 
   scope :preload_associations, -> { preload(:labels, project: :namespace) }
 
+  scope :public_only, -> { where(confidential: false) }
+
   after_save :expire_etag_cache
+  after_commit :update_project_counter_caches, on: :destroy
 
   attr_spammable :title, spam_title: true
   attr_spammable :description, spam_description: true
@@ -300,6 +303,10 @@ class Issue < ActiveRecord::Base
     end
   end
 
+  def update_project_counter_caches
+    Projects::OpenIssuesCountService.new(project).refresh_cache
+  end
+
   private
 
   # Returns `true` if the given User can read the current Issue.

app/models/merge_request.rb

@@ -34,6 +34,7 @@ class MergeRequest < ActiveRecord::Base
 
   after_create :ensure_merge_request_diff, unless: :importing?
   after_update :reload_diff_if_branch_changed
+  after_commit :update_project_counter_caches, on: :destroy
 
   # When this attribute is true some MR validation is ignored
   # It allows us to close or modify broken merge requests
@@ -707,9 +708,8 @@ class MergeRequest < ActiveRecord::Base
     if !include_description && closes_issues_references.present?
       message << "Closes #{closes_issues_references.to_sentence}"
     end
-
     message << "#{description}" if include_description && description.present?
-    message << "See merge request #{to_reference}"
+    message << "See merge request #{to_reference(full: true)}"
 
     message.join("\n\n")
   end
@@ -965,6 +965,10 @@ class MergeRequest < ActiveRecord::Base
     @base_pipeline ||= project.pipelines.find_by(sha: merge_request_diff&.base_commit_sha)
   end
 
+  def update_project_counter_caches
+    Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache
+  end
+
   private
 
   def write_ref

app/models/project.rb

@@ -251,6 +251,7 @@ class Project < ActiveRecord::Base
   scope :joined, ->(user) { where('namespace_id != ?', user.namespace_id) }
   scope :starred_by, ->(user) { joins(:users_star_projects).where('users_star_projects.user_id': user.id) }
   scope :visible_to_user, ->(user) { where(id: user.authorized_projects.select(:id).reorder(nil)) }
+  scope :archived, -> { where(archived: true) }
   scope :non_archived, -> { where(archived: false) }
   scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
   scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
@@ -1017,7 +1018,7 @@ class Project < ActiveRecord::Base
       name: name,
       description: description,
       web_url: web_url,
-      avatar_url: avatar_url,
+      avatar_url: avatar_url(only_path: false),
       git_ssh_url: ssh_url_to_repo,
       git_http_url: http_url_to_repo,
       namespace: namespace.name,
@@ -1167,7 +1168,11 @@ class Project < ActiveRecord::Base
   end
 
   def open_issues_count
-    issues.opened.count
+    Projects::OpenIssuesCountService.new(self).count
+  end
+
+  def open_merge_requests_count
+    Projects::OpenMergeRequestsCountService.new(self).count
   end
 
   def visibility_level_allowed_as_fork?(level = self.visibility_level)

app/models/repository.rb

@@ -213,12 +213,18 @@ class Repository
   end
 
   def branch_exists?(branch_name)
-    branch_names.include?(branch_name)
+    return false unless raw_repository
+
+    @branch_exists_memo ||= Hash.new do |hash, key|
+      hash[key] = raw_repository.branch_exists?(key)
+    end
+
+    @branch_exists_memo[branch_name]
   end
 
   def ref_exists?(ref)
-    rugged.references.exist?(ref)
-  rescue Rugged::ReferenceError
+    !!raw_repository&.ref_exists?(ref)
+  rescue ArgumentError
     false
   end
 
@@ -273,6 +279,7 @@ class Repository
   def expire_branches_cache
     expire_method_caches(%i(branch_names branch_count))
     @local_branches = nil
+    @branch_exists_memo = nil
   end
 
   def expire_statistics_caches

app/services/groups/nested_create_service.rb

+module Groups
+  class NestedCreateService < Groups::BaseService
+    attr_reader :group_path
+
+    def initialize(user, params)
+      @current_user, @params = user, params.dup
+
+      @group_path = @params.delete(:group_path)
+    end
+
+    def execute
+      return nil unless group_path
+
+      if group = Group.find_by_full_path(group_path)
+        return group
+      end
+
+      create_group_path
+    end
+
+    private
+
+    def create_group_path
+      group_path_segments = group_path.split('/')
+
+      last_group = nil
+      partial_path_segments = []
+      while subgroup_name = group_path_segments.shift
+        partial_path_segments << subgroup_name
+        partial_path = partial_path_segments.join('/')
+
+        new_params = params.reverse_merge(
+          path: subgroup_name,
+          name: subgroup_name,
+          parent: last_group
+        )
+        new_params[:visibility_level] ||= Gitlab::CurrentSettings.current_application_settings.default_group_visibility
+
+        last_group = Group.find_by_full_path(partial_path) || Groups::CreateService.new(current_user, new_params).execute
+      end
+
+      last_group
+    end
+  end
+end

app/services/issuable_base_service.rb

@@ -194,6 +194,8 @@ class IssuableBaseService < BaseService
 
   def after_create(issuable)
     # To be overridden by subclasses
+
+    issuable.update_project_counter_caches
   end
 
   def before_update(issuable)
@@ -202,6 +204,8 @@ class IssuableBaseService < BaseService
 
   def after_update(issuable)
     # To be overridden by subclasses
+
+    issuable.update_project_counter_caches
   end
 
   def update(issuable)

app/services/issues/create_service.rb

@@ -27,6 +27,8 @@ module Issues
       todo_service.new_issue(issuable, current_user)
       user_agent_detail_service.create
       resolve_discussions_with_issue(issuable)
+
+      super
     end
 
     def resolve_discussions_with_issue(issue)

app/services/merge_requests/create_service.rb

@@ -28,6 +28,8 @@ module MergeRequests
       todo_service.new_merge_request(issuable, current_user)
       issuable.cache_merge_request_closes_issues!(current_user)
       update_merge_requests_head_pipeline(issuable)
+
+      super
     end
 
     private

app/services/projects/count_service.rb

+module Projects
+  # Base class for the various service classes that count project data (e.g.
+  # issues or forks).
+  class CountService
+    def initialize(project)
+      @project = project
+    end
+
+    def relation_for_count
+      raise(
+        NotImplementedError,
+        '"relation_for_count" must be implemented and return an ActiveRecord::Relation'
+      )
+    end
+
+    def count
+      Rails.cache.fetch(cache_key) { uncached_count }
+    end
+
+    def refresh_cache
+      Rails.cache.write(cache_key, uncached_count)
+    end
+
+    def uncached_count
+      relation_for_count.count
+    end
+
+    def delete_cache
+      Rails.cache.delete(cache_key)
+    end
+
+    def cache_key_name
+      raise(
+        NotImplementedError,
+        '"cache_key_name" must be implemented and return a String'
+      )
+    end
+
+    def cache_key
+      ['projects', @project.id, cache_key_name]
+    end
+  end
+end

app/services/projects/forks_count_service.rb

 module Projects
   # Service class for getting and caching the number of forks of a project.
-  class ForksCountService
-    def initialize(project)
-      @project = project
+  class ForksCountService < CountService
+    def relation_for_count
+      @project.forks
     end
 
-    def count
-      Rails.cache.fetch(cache_key) { uncached_count }
-    end
-
-    def refresh_cache
-      Rails.cache.write(cache_key, uncached_count)
-    end
-
-    def delete_cache
-      Rails.cache.delete(cache_key)
-    end
-
-    private
-
-    def uncached_count
-      @project.forks.count
-    end
-
-    def cache_key
-      ['projects', @project.id, 'forks_count']
+    def cache_key_name
+      'forks_count'
     end
   end
 end

app/services/projects/open_issues_count_service.rb

+module Projects
+  # Service class for counting and caching the number of open issues of a
+  # project.
+  class OpenIssuesCountService < CountService
+    def relation_for_count
+      # We don't include confidential issues in this number since this would
+      # expose the number of confidential issues to non project members.
+      @project.issues.opened.public_only
+    end
+
+    def cache_key_name
+      'open_issues_count'
+    end
+  end
+end

app/services/projects/open_merge_requests_count_service.rb

+module Projects
+  # Service class for counting and caching the number of open merge requests of
+  # a project.
+  class OpenMergeRequestsCountService < CountService
+    def relation_for_count
+      @project.merge_requests.opened
+    end
+
+    def cache_key_name
+      'open_merge_requests_count'
+    end
+  end
+end

app/views/layouts/nav/_new_project_sidebar.html.haml

@@ -86,7 +86,8 @@
             %span.nav-item-name
               Issues
             - if @project.issues_enabled?
-              %span.badge.count.issue_counter= number_with_delimiter(IssuesFinder.new(current_user, project_id: @project.id).execute.opened.count)
+              %span.badge.count.issue_counter
+                = number_with_delimiter(@project.open_issues_count)
 
           %ul.sidebar-sub-level-items
             = nav_link(controller: :issues) do
@@ -116,7 +117,8 @@
               = custom_icon('mr_bold')
             %span.nav-item-name
               Merge Requests
-            %span.badge.count.merge_counter.js-merge-counter= number_with_delimiter(MergeRequestsFinder.new(current_user, project_id: @project.id).execute.opened.count)
+            %span.badge.count.merge_counter.js-merge-counter
+              = number_with_delimiter(@project.open_merge_requests_count)
 
       - if project_nav_tab? :pipelines
         = nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts]) do

app/views/layouts/nav/_project.html.haml

@@ -28,7 +28,8 @@
           %span
             Issues
             - if @project.issues_enabled?
-              %span.badge.count.issue_counter= number_with_delimiter(issuables_count_for_state(:issues, :opened, finder: IssuesFinder.new(current_user, project_id: @project.id)))
+              %span.badge.count.issue_counter
+                = number_with_delimiter(@project.open_issues_count)
 
     - if project_nav_tab? :merge_requests
       - controllers = [:merge_requests, 'projects/merge_requests/conflicts']
@@ -37,7 +38,8 @@
         = link_to project_merge_requests_path(@project), title: 'Merge Requests', class: 'shortcuts-merge_requests' do
           %span
             Merge Requests
-            %span.badge.count.merge_counter.js-merge-counter= number_with_delimiter(issuables_count_for_state(:merge_requests, :opened, finder: MergeRequestsFinder.new(current_user, project_id: @project.id)))
+            %span.badge.count.merge_counter.js-merge-counter
+              = number_with_delimiter(@project.open_merge_requests_count)
 
     - if project_nav_tab? :pipelines
       = nav_link(controller: [:pipelines, :builds, :environments, :artifacts]) do

app/views/projects/_project_templates.html.haml

 .project-templates-buttons.import-buttons{ data: { toggle: "buttons" } }
   .btn.blank-option.active
-    %input{ type: "radio",  autocomplete: "off", name: "project_templates", id: "blank", checked: "true" }
+    %input{ type: "radio",  autocomplete: "off", name: "project[template_name]", id: "blank", checked: "true", value: "" }
     = icon('file-o', class: 'btn-template-icon')
     Blank
   - Gitlab::ProjectTemplate.all.each do |template|

app/views/projects/commits/_commit.html.haml

@@ -41,14 +41,14 @@
 
 
       .commit-actions.hidden-xs
-        - if commit.status(ref)
-          = render_commit_status(commit, ref: ref)
-
         - if request.xhr?
           = render partial: 'projects/commit/signature', object: commit.signature
         - else
           = render partial: 'projects/commit/ajax_signature', locals: { commit: commit }
 
+        - if commit.status(ref)
+          = render_commit_status(commit, ref: ref)
+
         = link_to commit.short_id, project_commit_path(project, commit), class: "commit-sha btn btn-transparent"
         = clipboard_button(text: commit.id, title: _("Copy commit SHA to clipboard"))
         = link_to_browse_code(project, commit)

app/views/shared/projects/_dropdown.html.haml

@@ -15,8 +15,11 @@
       = link_to filter_projects_path(archived: nil), class: ("is-active" unless params[:archived].present?) do
         Hide archived projects
     %li
-      = link_to filter_projects_path(archived: true), class: ("is-active" if params[:archived].present?) do
+      = link_to filter_projects_path(archived: true), class: ("is-active" if Gitlab::Utils.to_boolean(params[:archived])) do
         Show archived projects
+    %li
+      = link_to filter_projects_path(archived: 'only'), class: ("is-active" if params[:archived] == 'only') do
+        Show archived projects only
     - if current_user
       %li.divider
       %li

changelogs/unreleased/13719-git-gc-timeout.yml

----
-title: "Raise Housekeeping timeout to 24 hours"
-merge_request: 13719

changelogs/unreleased/35994-archived-projects-only.yml

+---
+title: Add an option to list only archived projects
+merge_request: 13492
+author: Mehdi Lahmam (@mehlah)
+type: added

changelogs/unreleased/36262_merge_request_reference_in_merge_commit_global.yml

+---
+title: Merge request reference in merge commit changed to full reference
+merge_request: 13518
+author: haseebeqx
+type: fixed

changelogs/unreleased/bvl-improve-bare-project-import.yml

+---
+title: 'Improve bare project import: Allow subgroups, take default visibility level
+  into account'
+merge_request: 13670
+author:
+type: fixed

changelogs/unreleased/cache-issue-and-mr-counts.yml

+---
+title: Cache the number of open issues and merge requests
+merge_request:
+author:
+type: other

changelogs/unreleased/fix-old-mr-diffs.yml

+---
+title: Show un-highlighted text diffs when we do not have references to the correct
+  blobs
+merge_request:
+author:
+type: fixed

changelogs/unreleased/use_full_path_in_project_avatar_url_webhook.yml

+---
+title: Use full path of project's avatar in webhooks
+merge_request: 13649
+author: Vitaliy @blackst0ne Klachkov
+type: changed

changelogs/unreleased/zj-add-pipeline-source-variable.yml

+---
+title: Add CI_PIPELINE_SOURCE variable on CI Jobs
+merge_request:
+author:
+type: added

changelogs/unreleased/fix-broadcast-message-caching.yml → changelogs/unreleased/zj-fix-fe-blank-button.yml

 ---
-title: Fix caching of future broadcast messages
+title: Fix new project form not resetting the template value
 merge_request:
 author:
 type: fixed

doc/ci/variables/README.md

@@ -58,6 +58,7 @@ future GitLab releases.**
 | **CI_PIPELINE_ID**              | 8.10   | 0.5    | The unique id of the current pipeline that GitLab CI uses internally |
 | **CI_PIPELINE_SOURCE**          | 9.3    | all    | ([EEP]) The variable indicates how the pipeline was triggered, possible options are: push, web, trigger, schedule, api, pipeline |
 | **CI_PIPELINE_TRIGGERED**       | all    | all    | The flag to indicate that job was [triggered] |
+| **CI_PIPELINE_SOURCE**          | 10.0   | all    | The source for this pipeline, one of: push, web, trigger, schedule, api, external. Pipelines created before 9.5 will have unknown as source |
 | **CI_PROJECT_DIR**              | all    | all    | The full path where the repository is cloned and where the job is run |
 | **CI_PROJECT_ID**               | all    | all    | The unique id of the current project that GitLab CI uses internally |
 | **CI_PROJECT_NAME**             | 8.10   | 0.5    | The project name that is currently being built (actually it is project folder name) |

doc/development/licensing.md

@@ -66,7 +66,7 @@ Libraries with the following licenses are unacceptable for use:
 
 ## Requesting Approval for Licenses
 
-Libraries that are not listed in the [Acceptable Licenses][Acceptable-Licenses] or [Unacceptable Licenses][Unacceptable-Licenses] list can be submitted to the legal team for review. Please create an issue in the [Organization Repository][Org-Repo] and cc `@gl-legal`. After a decision has been made, the original requestor is responsible for updating this document.
+Libraries that are not listed in the [Acceptable Licenses][Acceptable-Licenses] or [Unacceptable Licenses][Unacceptable-Licenses] list can be submitted to the legal team for review. Please email `legal@gitlab.com` with the details. After a decision has been made, the original requestor is responsible for updating this document.
 
 ## Notes
 

lib/github/import.rb

@@ -107,7 +107,7 @@ module Github
       # this means that repo has wiki enabled, but have no pages. So,
       # we can skip the import.
       if e.message !~ /repository not exported/
-        errors(:wiki, wiki_url, e.message)
+        error(:wiki, wiki_url, e.message)
       end
     end
 

lib/gitlab/bare_repository_importer.rb

+module Gitlab
+  class BareRepositoryImporter
+    NoAdminError = Class.new(StandardError)
+
+    def self.execute
+      Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
+        git_base_path = repository_storage['path']
+        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
+
+        repos_to_import.each do |repo_path|
+          if repo_path.end_with?('.wiki.git')
+            log " * Skipping wiki repo"
+            next
+          end
+
+          log "Processing #{repo_path}".color(:yellow)
+
+          repo_relative_path = repo_path[repository_storage['path'].length..-1]
+                                 .sub(/^\//, '') # Remove leading `/`
+                                 .sub(/\.git$/, '') # Remove `.git` at the end
+          new(storage_name, repo_relative_path).create_project_if_needed
+        end
+      end
+    end
+
+    attr_reader :storage_name, :full_path, :group_path, :project_path, :user
+    delegate :log, to: :class
+
+    def initialize(storage_name, repo_path)
+      @storage_name = storage_name
+      @full_path = repo_path
+
+      unless @user = User.admins.order_id_asc.first
+        raise NoAdminError.new('No admin user found to import repositories')
+      end
+
+      @group_path, @project_path = File.split(repo_path)
+      @group_path = nil if @group_path == '.'
+    end
+
+    def create_project_if_needed
+      if project = Project.find_by_full_path(full_path)
+        log " * #{project.name} (#{full_path}) exists"
+        return project
+      end
+
+      create_project
+    end
+
+    private
+
+    def create_project
+      group = find_or_create_group
+
+      project_params = {
+        name: project_path,
+        path: project_path,
+        repository_storage: storage_name,
+        namespace_id: group&.id
+      }
+
+      project = Projects::CreateService.new(user, project_params).execute
+
+      if project.persisted?
+        log " * Created #{project.name} (#{full_path})".color(:green)
+        ProjectCacheWorker.perform_async(project.id)
+      else
+        log " * Failed trying to create #{project.name} (#{full_path})".color(:red)
+        log "   Errors: #{project.errors.messages}".color(:red)
+      end
+
+      project
+    end
+
+    def find_or_create_group
+      return nil unless group_path
+
+      if namespace = Namespace.find_by_full_path(group_path)
+        log " * Namespace #{group_path} exists.".color(:green)
+        return namespace
+      end
+
+      log " * Creating Group: #{group_path}"
+      Groups::NestedCreateService.new(user, group_path: group_path).execute
+    end
+
+    # This is called from within a rake task only used by Admins, so allow writing
+    # to STDOUT
+    #
+    # rubocop:disable Rails/Output
+    def self.log(message)
+      puts message
+    end
+    # rubocop:enable Rails/Output
+  end
+end

lib/gitlab/diff/file.rb

@@ -186,7 +186,10 @@ module Gitlab
       end
 
       def content_changed?
-        old_blob && new_blob && old_blob.id != new_blob.id
+        return blobs_changed? if diff_refs
+        return false if new_file? || deleted_file? || renamed_file?
+
+        text? && diff_lines.any?
       end
 
       def different_type?
@@ -225,6 +228,10 @@ module Gitlab
 
       private
 
+      def blobs_changed?
+        old_blob && new_blob && old_blob.id != new_blob.id
+      end
+
       def simple_viewer_class
         return DiffViewer::NotDiffable unless diffable?
 
@@ -250,6 +257,8 @@ module Gitlab
           DiffViewer::Renamed
         elsif mode_changed?
           DiffViewer::ModeChanged
+        else
+          DiffViewer::NoPreview
         end
       end
 

lib/gitlab/git/repository.rb

@@ -153,7 +153,7 @@ module Gitlab
           if is_enabled
             gitaly_ref_client.count_branch_names
           else
-            rugged.branches.count do |ref|
+            rugged.branches.each(:local).count do |ref|
               begin
                 ref.name && ref.target # ensures the branch is valid
 
@@ -201,6 +201,19 @@ module Gitlab
         end
       end
 
+      # Returns true if the given ref name exists
+      #
+      # Ref names must start with `refs/`.
+      def ref_exists?(ref_name)
+        gitaly_migrate(:ref_exists) do |is_enabled|
+          if is_enabled
+            gitaly_ref_exists?(ref_name)
+          else
+            rugged_ref_exists?(ref_name)
+          end
+        end
+      end
+
       # Returns true if the given tag exists
       #
       # name - The name of the tag as a String.
@@ -989,6 +1002,16 @@ module Gitlab
         raw_output.compact
       end
 
+      # Returns true if the given ref name exists
+      #
+      # Ref names must start with `refs/`.
+      def rugged_ref_exists?(ref_name)
+        raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/')
+        rugged.references.exist?(ref_name)
+      rescue Rugged::ReferenceError
+        false
+      end
+
       # Returns true if the given ref name exists
       #
       # Ref names must start with `refs/`.

lib/gitlab/gitaly_client/commit_service.rb

@@ -175,8 +175,8 @@ module Gitlab
       def raw_blame(revision, path)
         request = Gitaly::RawBlameRequest.new(
           repository: @gitaly_repo,
-          revision: revision,
-          path: path
+          revision: GitalyClient.encode(revision),
+          path: GitalyClient.encode(path)
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request)

lib/gitlab/gitaly_client/ref_service.rb

@@ -71,7 +71,7 @@ module Gitlab
       end
 
       def ref_exists?(ref_name)
-        request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: ref_name)
+        request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name))
         response = GitalyClient.call(@storage, :ref_service, :ref_exists, request)
         response.value
       rescue GRPC::InvalidArgument => e

lib/tasks/gitlab/import.rake

@@ -9,6 +9,7 @@ namespace :gitlab do
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
     #
+    #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
       if Project.current_application_settings.hashed_storage_enabled
@@ -17,69 +18,7 @@ namespace :gitlab do
         exit 1
       end
 
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        git_base_path = repository_storage['path']
-        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
-
-        repos_to_import.each do |repo_path|
-          # strip repo base path
-          repo_path[0..git_base_path.length] = ''
-
-          path = repo_path.sub(/\.git$/, '')
-          group_name, name = File.split(path)
-          group_name = nil if group_name == '.'
-
-          puts "Processing #{repo_path}".color(:yellow)
-
-          if path.end_with?('.wiki')
-            puts " * Skipping wiki repo"
-            next
-          end
-
-          project = Project.find_by_full_path(path)
-
-          if project
-            puts " * #{project.name} (#{repo_path}) exists"
-          else
-            user = User.admins.reorder("id").first
-
-            project_params = {
-              name: name,
-              path: name
-            }
-
-            # find group namespace
-            if group_name
-              group = Namespace.find_by(path: group_name)
-              # create group namespace
-              unless group
-                group = Group.new(name: group_name)
-                group.path = group_name
-                group.owner = user
-                if group.save
-                  puts " * Created Group #{group.name} (#{group.id})".color(:green)
-                else
-                  puts " * Failed trying to create group #{group.name}".color(:red)
-                end
-              end
-              # set project group
-              project_params[:namespace_id] = group.id
-            end
-
-            project = Projects::CreateService.new(user, project_params).execute
-
-            if project.persisted?
-              puts " * Created #{project.name} (#{repo_path})".color(:green)
-              ProjectCacheWorker.perform_async(project.id)
-            else
-              puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
-              puts "   Errors: #{project.errors.messages}".color(:red)
-            end
-          end
-        end
-      end
-
-      puts "Done!".color(:green)
+      Gitlab::BareRepositoryImporter.execute
     end
   end
 end

lib/tasks/import.rake

@@ -72,23 +72,7 @@ class GithubImport
     return @current_user.namespace if names == @current_user.namespace_path
     return @current_user.namespace unless @current_user.can_create_group?
 
-    full_path_namespace = Namespace.find_by_full_path(names)
-
-    return full_path_namespace if full_path_namespace
-
-    names.split('/').inject(nil) do |parent, name|
-      begin
-        namespace = Group.create!(name: name,
-                                  path: name,
-                                  owner: @current_user,
-                                  parent: parent)
-        namespace.add_owner(@current_user)
-
-        namespace
-      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
-        Namespace.where(parent: parent).find_by_path_or_name(name)
-      end
-    end
+    Groups::NestedCreateService.new(@current_user, group_path: names).execute
   end
 
   def full_path_namespace(names)

spec/features/admin/admin_projects_spec.rb

@@ -36,6 +36,14 @@ describe "Admin::Projects"  do
       expect(page).to have_content(archived_project.name)
       expect(page).to have_xpath("//span[@class='label label-warning']", text: 'archived')
     end
+
+    it 'renders only archived projects', js: true do
+      find(:css, '#sort-projects-dropdown').click
+      click_link 'Show archived projects only'
+
+      expect(page).to have_content(archived_project.name)
+      expect(page).not_to have_content(project.name)
+    end
   end
 
   describe "GET /admin/projects/:namespace_id/:id" do

spec/features/calendar_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 feature 'Contributions Calendar', :js do
   let(:user) { create(:user) }
-  let(:contributed_project) { create(:project, :public) }
+  let(:contributed_project) { create(:project, :public, :repository) }
   let(:issue_note) { create(:note, project: contributed_project) }
 
   # Ex/ Sunday Jan 1, 2016

spec/features/dashboard/activity_spec.rb

@@ -17,7 +17,7 @@ feature 'Dashboard > Activity' do
   end
 
   context 'event filters', :js do
-    let(:project) { create(:project) }
+    let(:project) { create(:project, :repository) }
 
     let(:merge_request) do
       create(:merge_request, author: user, source_project: project, target_project: project)

spec/features/dashboard/archived_projects_spec.rb

@@ -26,6 +26,13 @@ RSpec.describe 'Dashboard Archived Project' do
     expect(page).to have_link(archived_project.name)
   end
 
+  it 'renders only archived projects' do
+    click_link 'Show archived projects only'
+
+    expect(page).to have_content(archived_project.name)
+    expect(page).not_to have_content(project.name)
+  end
+
   it 'searchs archived projects', :js do
     click_button 'Last updated'
     click_link 'Show archived projects'

spec/features/merge_requests/merge_commit_message_toggle_spec.rb

@@ -19,7 +19,7 @@ feature 'Clicking toggle commit message link', js: true do
       "Merge branch 'feature' into 'master'",
       merge_request.title,
       "Closes #{issue_1.to_reference} and #{issue_2.to_reference}",
-      "See merge request #{merge_request.to_reference}"
+      "See merge request #{merge_request.to_reference(full: true)}"
     ].join("\n\n")
   end
   let(:message_with_description) do
@@ -27,7 +27,7 @@ feature 'Clicking toggle commit message link', js: true do
       "Merge branch 'feature' into 'master'",
       merge_request.title,
       merge_request.description,
-      "See merge request #{merge_request.to_reference}"
+      "See merge request #{merge_request.to_reference(full: true)}"
     ].join("\n\n")
   end
 

spec/finders/admin/projects_finder_spec.rb

@@ -118,6 +118,12 @@ describe Admin::ProjectsFinder do
 
         it { is_expected.to match_array([archived_project, shared_project, public_project, internal_project, private_project]) }
       end
+
+      context 'archived=only' do
+        let(:params) { { archived: 'only' } }
+
+        it { is_expected.to eq([archived_project]) }
+      end
     end
 
     context 'filter by personal' do

spec/finders/projects_finder_spec.rb

@@ -123,6 +123,12 @@ describe ProjectsFinder do
         it { is_expected.to match_array([public_project, internal_project, archived_project]) }
       end
 
+      describe 'filter by archived only' do
+        let(:params) { { archived: 'only' } }
+
+        it { is_expected.to eq([archived_project]) }
+      end
+
       describe 'filter by archived for backward compatibility' do
         let(:params) { { archived: false } }
 

spec/lib/gitlab/bare_repository_importer_spec.rb

+require 'spec_helper'
+
+describe Gitlab::BareRepositoryImporter, repository: true do
+  subject(:importer) { described_class.new('default', project_path) }
+  let(:project_path) { 'a-group/a-sub-group/a-project' }
+  let!(:admin) { create(:admin) }
+
+  before do
+    allow(described_class).to receive(:log)
+  end
+
+  describe '.execute' do
+    it 'creates a project for a repository in storage' do
+      FileUtils.mkdir_p(File.join(TestEnv.repos_path, "#{project_path}.git"))
+      fake_importer = double
+
+      expect(described_class).to receive(:new).with('default', project_path)
+                                   .and_return(fake_importer)
+      expect(fake_importer).to receive(:create_project_if_needed)
+
+      described_class.execute
+    end
+
+    it 'skips wiki repos' do
+      FileUtils.mkdir_p(File.join(TestEnv.repos_path, 'the-group', 'the-project.wiki.git'))
+
+      expect(described_class).to receive(:log).with(' * Skipping wiki repo')
+      expect(described_class).not_to receive(:new)
+
+      described_class.execute
+    end
+  end
+
+  describe '#initialize' do
+    context 'without admin users' do
+      let(:admin) { nil }
+
+      it 'raises an error' do
+        expect { importer }.to raise_error(Gitlab::BareRepositoryImporter::NoAdminError)
+      end
+    end
+  end
+
+  describe '#create_project_if_needed' do
+    it 'starts an import for a project that did not exist' do
+      expect(importer).to receive(:create_project)
+
+      importer.create_project_if_needed
+    end
+
+    it 'skips importing when the project already exists' do
+      group = create(:group, path: 'a-group')
+      subgroup = create(:group, path: 'a-sub-group', parent: group)
+      project = create(:project, path: 'a-project', namespace: subgroup)
+
+      expect(importer).not_to receive(:create_project)
+      expect(importer).to receive(:log).with(" * #{project.name} (a-group/a-sub-group/a-project) exists")
+
+      importer.create_project_if_needed
+    end
+
+    it 'creates a project with the correct path in the database' do
+      importer.create_project_if_needed
+
+      expect(Project.find_by_full_path(project_path)).not_to be_nil
+    end
+  end
+end

spec/lib/gitlab/diff/file_spec.rb

@@ -15,6 +15,17 @@ describe Gitlab::Diff::File do
     it { expect(diff_lines.first).to be_kind_of(Gitlab::Diff::Line) }
   end
 
+  describe '#highlighted_diff_lines' do
+    it 'highlights the diff and memoises the result' do
+      expect(Gitlab::Diff::Highlight).to receive(:new)
+                                           .with(diff_file, repository: project.repository)
+                                           .once
+                                           .and_call_original
+
+      diff_file.highlighted_diff_lines
+    end
+  end
+
   describe '#mode_changed?' do
     it { expect(diff_file.mode_changed?).to be_falsey }
   end
@@ -122,8 +133,20 @@ describe Gitlab::Diff::File do
         let(:commit) { project.commit('2f63565e7aac07bcdadb654e253078b727143ec4') }
         let(:diff_file) { commit.diffs.diff_file_with_new_path('files/images/6049019_460s.jpg') }
 
-        it 'returns true' do
-          expect(diff_file.content_changed?).to be_truthy
+        context 'when the blobs are different' do
+          it 'returns true' do
+            expect(diff_file.content_changed?).to be_truthy
+          end
+        end
+
+        context 'when there are no diff refs' do
+          before do
+            allow(diff_file).to receive(:diff_refs).and_return(nil)
+          end
+
+          it 'returns false' do
+            expect(diff_file.content_changed?).to be_falsey
+          end
         end
       end
 
@@ -131,8 +154,20 @@ describe Gitlab::Diff::File do
         let(:commit) { project.commit('570e7b2abdd848b95f2f578043fc23bd6f6fd24d') }
         let(:diff_file) { commit.diffs.diff_file_with_new_path('files/ruby/popen.rb') }
 
-        it 'returns true' do
-          expect(diff_file.content_changed?).to be_truthy
+        context 'when the blobs are different' do
+          it 'returns true' do
+            expect(diff_file.content_changed?).to be_truthy
+          end
+        end
+
+        context 'when there are no diff refs' do
+          before do
+            allow(diff_file).to receive(:diff_refs).and_return(nil)
+          end
+
+          it 'returns true' do
+            expect(diff_file.content_changed?).to be_truthy
+          end
         end
       end
     end
@@ -270,6 +305,21 @@ describe Gitlab::Diff::File do
         expect(diff_file.simple_viewer).to be_a(DiffViewer::ModeChanged)
       end
     end
+
+    context 'when no other conditions apply' do
+      before do
+        allow(diff_file).to receive(:content_changed?).and_return(false)
+        allow(diff_file).to receive(:new_file?).and_return(false)
+        allow(diff_file).to receive(:deleted_file?).and_return(false)
+        allow(diff_file).to receive(:renamed_file?).and_return(false)
+        allow(diff_file).to receive(:mode_changed?).and_return(false)
+        allow(diff_file).to receive(:raw_text?).and_return(false)
+      end
+
+      it 'returns a No Preview viewer' do
+        expect(diff_file.simple_viewer).to be_a(DiffViewer::NoPreview)
+      end
+    end
   end
 
   describe '#rich_viewer' do

spec/lib/gitlab/git/repository_spec.rb

@@ -977,6 +977,36 @@ describe Gitlab::Git::Repository, seed_helper: true do
     it 'returns the number of branches' do
       expect(repository.branch_count).to eq(10)
     end
+
+    context 'with local and remote branches' do
+      let(:repository) do
+        Gitlab::Git::Repository.new('default', File.join(TEST_MUTABLE_REPO_PATH, '.git'))
+      end
+
+      before do
+        create_remote_branch(repository, 'joe', 'remote_branch', 'master')
+        repository.create_branch('local_branch', 'master')
+      end
+
+      after do
+        FileUtils.rm_rf(TEST_MUTABLE_REPO_PATH)
+        ensure_seeds
+      end
+
+      it 'returns the count of local branches' do
+        expect(repository.branch_count).to eq(repository.local_branches.count)
+      end
+
+      context 'with Gitaly disabled' do
+        before do
+          allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(false)
+        end
+
+        it 'returns the count of local branches' do
+          expect(repository.branch_count).to eq(repository.local_branches.count)
+        end
+      end
+    end
   end
 
   describe "#ls_files" do
@@ -1080,6 +1110,34 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#ref_exists?' do
+    shared_examples 'checks the existence of refs' do
+      it 'returns true for an existing tag' do
+        expect(repository.ref_exists?('refs/heads/master')).to eq(true)
+      end
+
+      it 'returns false for a non-existing tag' do
+        expect(repository.ref_exists?('refs/tags/THIS_TAG_DOES_NOT_EXIST')).to eq(false)
+      end
+
+      it 'raises an ArgumentError for an empty string' do
+        expect { repository.ref_exists?('') }.to raise_error(ArgumentError)
+      end
+
+      it 'raises an ArgumentError for an invalid ref' do
+        expect { repository.ref_exists?('INVALID') }.to raise_error(ArgumentError)
+      end
+    end
+
+    context 'when Gitaly ref_exists feature is enabled' do
+      it_behaves_like 'checks the existence of refs'
+    end
+
+    context 'when Gitaly ref_exists feature is disabled', skip_gitaly_mock: true do
+      it_behaves_like 'checks the existence of refs'
+    end
+  end
+
   describe '#tag_exists?' do
     shared_examples 'checks the existence of tags' do
       it 'returns true for an existing tag' do

spec/models/ci/pipeline_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Ci::Pipeline, :mailer do
   let(:user) { create(:user) }
-  let(:project) { create(:project) }
+  set(:project) { create(:project) }
 
   let(:pipeline) do
     create(:ci_empty_pipeline, status: :created, project: project)
@@ -163,6 +163,18 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '#predefined_variables' do
+    subject { pipeline.predefined_variables }
+
+    it { is_expected.to be_an(Array) }
+
+    it 'includes the defined keys' do
+      keys = subject.map { |v| v[:key] }
+
+      expect(keys).to include('CI_PIPELINE_ID', 'CI_CONFIG_PATH', 'CI_PIPELINE_SOURCE')
+    end
+  end
+
   describe '#auto_canceled?' do
     subject { pipeline.auto_canceled? }
 

spec/models/issue_spec.rb

@@ -772,4 +772,22 @@ describe Issue do
       end
     end
   end
+
+  describe 'removing an issue' do
+    it 'refreshes the number of open issues of the project' do
+      project = subject.project
+
+      expect { subject.destroy }
+        .to change { project.open_issues_count }.from(1).to(0)
+    end
+  end
+
+  describe '.public_only' do
+    it 'only returns public issues' do
+      public_issue = create(:issue)
+      create(:issue, confidential: true)
+
+      expect(described_class.public_only).to eq([public_issue])
+    end
+  end
 end

spec/models/merge_request_spec.rb

@@ -813,7 +813,7 @@ describe MergeRequest do
       request = build_stubbed(:merge_request)
 
       expect(request.merge_commit_message)
-        .to match("See merge request #{request.to_reference}")
+        .to match("See merge request #{request.to_reference(full: true)}")
     end
 
     it 'excludes multiple linebreak runs when description is blank' do
@@ -2179,4 +2179,13 @@ describe MergeRequest do
       expect(subject.ref_fetched?).to be_falsey
     end
   end
+
+  describe 'removing a merge request' do
+    it 'refreshes the number of open merge requests of the target project' do
+      project = subject.target_project
+
+      expect { subject.destroy }
+        .to change { project.open_merge_requests_count }.from(1).to(0)
+    end
+  end
 end

spec/models/user_spec.rb

@@ -1424,7 +1424,7 @@ describe User do
     end
 
     it "excludes push event if branch has been deleted" do
-      allow_any_instance_of(Repository).to receive(:branch_names).and_return(['foo'])
+      allow_any_instance_of(Repository).to receive(:branch_exists?).with('master').and_return(false)
 
       expect(subject.recent_push).to eq(nil)
     end

spec/services/groups/nested_create_service_spec.rb

+require 'spec_helper'
+
+describe Groups::NestedCreateService do
+  let(:user) { create(:user) }
+  let(:params) { { group_path: 'a-group/a-sub-group' } }
+
+  subject(:service) { described_class.new(user, params) }
+
+  describe "#execute" do
+    it 'returns the group if it already existed' do
+      parent = create(:group, path: 'a-group', owner: user)
+      child = create(:group, path: 'a-sub-group', parent: parent, owner: user)
+
+      expect(service.execute).to eq(child)
+    end
+
+    it 'reuses a parent if it already existed' do
+      parent = create(:group, path: 'a-group')
+      parent.add_owner(user)
+
+      expect(service.execute.parent).to eq(parent)
+    end
+
+    it 'creates group and subgroup in the database' do
+      service.execute
+
+      parent = Group.find_by_full_path('a-group')
+      child = parent.children.find_by(path: 'a-sub-group')
+
+      expect(parent).not_to be_nil
+      expect(child).not_to be_nil
+    end
+
+    it 'creates the group with correct visibility level' do
+      allow(Gitlab::CurrentSettings.current_application_settings)
+        .to receive(:default_group_visibility) { Gitlab::VisibilityLevel::INTERNAL }
+
+      group = service.execute
+
+      expect(group.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL)
+    end
+
+    context 'adding a visibility level ' do
+      let(:params) { { group_path: 'a-group/a-sub-group', visibility_level: Gitlab::VisibilityLevel::PRIVATE } }
+
+      it 'overwrites the visibility level' do
+        group = service.execute
+
+        expect(group.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
+      end
+    end
+  end
+end

spec/services/issues/close_service_spec.rb

@@ -42,6 +42,11 @@ describe Issues::CloseService do
       service.execute(issue)
     end
 
+    it 'refreshes the number of open issues' do
+      expect { service.execute(issue) }
+        .to change { project.open_issues_count }.from(1).to(0)
+    end
+
     it 'invalidates counter cache for assignees' do
       expect_any_instance_of(User).to receive(:invalidate_issue_cache_counts)
 

spec/services/issues/create_service_spec.rb

@@ -35,6 +35,10 @@ describe Issues::CreateService do
         expect(issue.due_date).to eq Date.tomorrow
       end
 
+      it 'refreshes the number of open issues' do
+        expect { issue }.to change { project.open_issues_count }.from(0).to(1)
+      end
+
       context 'when current user cannot admin issues in the project' do
         let(:guest) { create(:user) }
 

spec/services/issues/reopen_service_spec.rb

@@ -34,6 +34,13 @@ describe Issues::ReopenService do
         described_class.new(project, user).execute(issue)
       end
 
+      it 'refreshes the number of opened issues' do
+        service = described_class.new(project, user)
+
+        expect { service.execute(issue) }
+          .to change { project.open_issues_count }.from(0).to(1)
+      end
+
       context 'when issue is not confidential' do
         it 'executes issue hooks' do
           expect(project).to receive(:execute_hooks).with(an_instance_of(Hash), :issue_hooks)

spec/services/merge_requests/close_service_spec.rb

@@ -52,6 +52,13 @@ describe MergeRequests::CloseService do
       end
     end
 
+    it 'refreshes the number of open merge requests for a valid MR' do
+      service = described_class.new(project, user, {})
+
+      expect { service.execute(merge_request) }
+        .to change { project.open_merge_requests_count }.from(1).to(0)
+    end
+
     context 'current user is not authorized to close merge request' do
       before do
         perform_enqueued_jobs do

spec/services/merge_requests/create_service_spec.rb

@@ -18,31 +18,35 @@ describe MergeRequests::CreateService do
       end
 
       let(:service) { described_class.new(project, user, opts) }
+      let(:merge_request) { service.execute }
 
       before do
         project.team << [user, :master]
         project.team << [assignee, :developer]
         allow(service).to receive(:execute_hooks)
-
-        @merge_request = service.execute
       end
 
       it 'creates an MR' do
-        expect(@merge_request).to be_valid
-        expect(@merge_request.title).to eq('Awesome merge_request')
-        expect(@merge_request.assignee).to be_nil
-        expect(@merge_request.merge_params['force_remove_source_branch']).to eq('1')
+        expect(merge_request).to be_valid
+        expect(merge_request.title).to eq('Awesome merge_request')
+        expect(merge_request.assignee).to be_nil
+        expect(merge_request.merge_params['force_remove_source_branch']).to eq('1')
       end
 
       it 'executes hooks with default action' do
-        expect(service).to have_received(:execute_hooks).with(@merge_request)
+        expect(service).to have_received(:execute_hooks).with(merge_request)
+      end
+
+      it 'refreshes the number of open merge requests' do
+        expect { service.execute }
+          .to change { project.open_merge_requests_count }.from(0).to(1)
       end
 
       it 'does not creates todos' do
         attributes = {
           project: project,
-          target_id: @merge_request.id,
-          target_type: @merge_request.class.name
+          target_id: merge_request.id,
+          target_type: merge_request.class.name
         }
 
         expect(Todo.where(attributes).count).to be_zero
@@ -51,8 +55,8 @@ describe MergeRequests::CreateService do
       it 'creates exactly 1 create MR event' do
         attributes = {
           action: Event::CREATED,
-          target_id: @merge_request.id,
-          target_type: @merge_request.class.name
+          target_id: merge_request.id,
+          target_type: merge_request.class.name
         }
 
         expect(Event.where(attributes).count).to eq(1)
@@ -69,15 +73,15 @@ describe MergeRequests::CreateService do
           }
         end
 
-        it { expect(@merge_request.assignee).to eq assignee }
+        it { expect(merge_request.assignee).to eq assignee }
 
         it 'creates a todo for new assignee' do
           attributes = {
             project: project,
             author: user,
             user: assignee,
-            target_id: @merge_request.id,
-            target_type: @merge_request.class.name,
+            target_id: merge_request.id,
+            target_type: merge_request.class.name,
             action: Todo::ASSIGNED,
             state: :pending
           }

spec/services/merge_requests/reopen_service_spec.rb

@@ -47,6 +47,13 @@ describe MergeRequests::ReopenService do
       end
     end
 
+    it 'refreshes the number of open merge requests for a valid MR' do
+      service = described_class.new(project, user, {})
+
+      expect { service.execute(merge_request) }
+        .to change { project.open_merge_requests_count }.from(0).to(1)
+    end
+
     context 'current user is not authorized to reopen merge request' do
       before do
         perform_enqueued_jobs do

spec/services/projects/count_service_spec.rb

+require 'spec_helper'
+
+describe Projects::CountService do
+  let(:project) { build(:project, id: 1) }
+  let(:service) { described_class.new(project) }
+
+  describe '#relation_for_count' do
+    it 'raises NotImplementedError' do
+      expect { service.relation_for_count }.to raise_error(NotImplementedError)
+    end
+  end
+
+  describe '#count' do
+    before do
+      allow(service).to receive(:cache_key_name).and_return('count_service')
+    end
+
+    it 'returns the number of rows' do
+      allow(service).to receive(:uncached_count).and_return(1)
+
+      expect(service.count).to eq(1)
+    end
+
+    it 'caches the number of rows', :use_clean_rails_memory_store_caching do
+      expect(service).to receive(:uncached_count).once.and_return(1)
+
+      2.times do
+        expect(service.count).to eq(1)
+      end
+    end
+  end
+
+  describe '#refresh_cache', :use_clean_rails_memory_store_caching do
+    before do
+      allow(service).to receive(:cache_key_name).and_return('count_service')
+    end
+
+    it 'refreshes the cache' do
+      expect(service).to receive(:uncached_count).once.and_return(1)
+
+      service.refresh_cache
+
+      expect(service.count).to eq(1)
+    end
+  end
+
+  describe '#delete_cache', :use_clean_rails_memory_store_caching do
+    before do
+      allow(service).to receive(:cache_key_name).and_return('count_service')
+    end
+
+    it 'removes the cache' do
+      expect(service).to receive(:uncached_count).twice.and_return(1)
+
+      service.count
+      service.delete_cache
+      service.count
+    end
+  end
+
+  describe '#cache_key_name' do
+    it 'raises NotImplementedError' do
+      expect { service.cache_key_name }.to raise_error(NotImplementedError)
+    end
+  end
+
+  describe '#cache_key' do
+    it 'returns the cache key as an Array' do
+      allow(service).to receive(:cache_key_name).and_return('count_service')
+      expect(service.cache_key).to eq(['projects', 1, 'count_service'])
+    end
+  end
+end

spec/services/projects/forks_count_service_spec.rb

 require 'spec_helper'
 
 describe Projects::ForksCountService do
-  let(:project) { build(:project, id: 42) }
-  let(:service) { described_class.new(project) }
-
   describe '#count' do
     it 'returns the number of forks' do
-      allow(service).to receive(:uncached_count).and_return(1)
-
-      expect(service.count).to eq(1)
-    end
-
-    it 'caches the forks count', :use_clean_rails_memory_store_caching do
-      expect(service).to receive(:uncached_count).once.and_return(1)
+      project = build(:project, id: 42)
+      service = described_class.new(project)
 
-      2.times { service.count }
-    end
-  end
-
-  describe '#refresh_cache', :use_clean_rails_memory_store_caching do
-    it 'refreshes the cache' do
-      expect(service).to receive(:uncached_count).once.and_return(1)
-
-      service.refresh_cache
+      allow(service).to receive(:uncached_count).and_return(1)
 
       expect(service.count).to eq(1)
     end
   end
-
-  describe '#delete_cache', :use_clean_rails_memory_store_caching do
-    it 'removes the cache' do
-      expect(service).to receive(:uncached_count).twice.and_return(1)
-
-      service.count
-      service.delete_cache
-      service.count
-    end
-  end
 end

spec/services/projects/open_issues_count_service_spec.rb

+require 'spec_helper'
+
+describe Projects::OpenIssuesCountService do
+  describe '#count' do
+    it 'returns the number of open issues' do
+      project = create(:project)
+      create(:issue, :opened, project: project)
+
+      expect(described_class.new(project).count).to eq(1)
+    end
+
+    it 'does not include confidential issues in the issue count' do
+      project = create(:project)
+
+      create(:issue, :opened, project: project)
+      create(:issue, :opened, confidential: true, project: project)
+
+      expect(described_class.new(project).count).to eq(1)
+    end
+  end
+end

spec/services/projects/open_merge_requests_count_service_spec.rb

+require 'spec_helper'
+
+describe Projects::OpenMergeRequestsCountService do
+  describe '#count' do
+    it 'returns the number of open merge requests' do
+      project = create(:project)
+      create(:merge_request,
+             :opened,
+             source_project: project,
+             target_project: project)
+
+      expect(described_class.new(project).count).to eq(1)
+    end
+  end
+end

spec/support/db_cleaner.rb

@@ -5,7 +5,7 @@ RSpec.configure do |config|
   end
 
   config.append_after(:context) do
-    DatabaseCleaner.clean_with(:truncation)
+    DatabaseCleaner.clean_with(:truncation, cache_tables: false)
   end
 
   config.before(:each) do
@@ -13,11 +13,11 @@ RSpec.configure do |config|
     DatabaseCleaner.strategy = :transaction
   end
 
-  config.before(:each, js: true) do
+  config.before(:each, :js) do
     DatabaseCleaner.strategy = :truncation, { except: ['licenses'] }
   end
 
-  config.before(:each, truncate: true) do
+  config.before(:each, :truncate) do
     DatabaseCleaner.strategy = :truncation, { except: ['licenses'] }
   end