Merge remote-tracking branch 'ee/master' into ce-to-ee-2017-09-29

* ee/master:
  (EE) Create repositories via Gitaly
  Remove changelogs with invalid extensions
  Extend changelog checker to test file extensions
  Fix invalid changelog entries
  Manually add CHANGELOG entry for !13325
  Add static analysis job to find invalid YAML in changelogs
  Merge branch 'sh-fix-gitlab-qa-admin' into 'master'
  Adds a Gitaly n+1 notice to commit handling on access check
  Fix EE delta size check handling with annotated tags
  Makes code quality requests in parallel
  Fix Ruby code for disabling Auto DevOps banner

CHANGELOG.md

@@ -85,6 +85,8 @@ entry.
 - [FIXED] Fixed merge request changes bar jumping.
 - [FIXED] Improve migrations using triggers.
 - [FIXED] Fix ConvDev Index nav item and Monitoring submenu regression.
+- [FIXED] disabling notifications globally now properly turns off group/project added
+  emails !13325
 - [DEPRECATED] Deprecate custom SSH client configuration for the git user. !13930
 - [CHANGED] allow all users to delete their account. !13636 (Jacopo Beschi @jacopo-beschi)
 - [CHANGED] Use full path of project's avatar in webhooks. !13649 (Vitaliy @blackst0ne Klachkov)

Gemfile

@@ -414,7 +414,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.37.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.38.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -299,7 +299,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.37.0)
+    gitaly-proto (0.38.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1060,7 +1060,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.37.0)
+  gitaly-proto (~> 0.38.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)

app/models/project.rb

@@ -1035,7 +1035,7 @@ class Project < ActiveRecord::Base
     # Forked import is handled asynchronously
     return if forked? && !force
 
-    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+    if gitlab_shell.add_repository(repository_storage, disk_path)
       repository.after_create
       true
     else

app/models/project_wiki.rb

@@ -189,7 +189,7 @@ class ProjectWiki
   private
 
   def init_repo(disk_path)
-    gitlab_shell.add_repository(project.repository_storage_path, disk_path)
+    gitlab_shell.add_repository(project.repository_storage, disk_path)
   end
 
   def commit_details(action, message = nil, title = nil)

changelogs/unreleased-ee/25011-hoverstates-for-collapsed-issue-merge-request-sidebar-EE

----
-title: Adds hoverstates for collapsed Issue/Merge Request sidebar for Time tracking Icon
-merge_request:
-author:

changelogs/unreleased-ee/sh-delta-check-with-annotated-tags.yml

+---
+title: Fix EE delta size check handling with annotated tags
+merge_request:
+author:
+type: fixed

changelogs/unreleased/12673-fix_v3_project_hooks_build_events

----
-title: "Fix v3 api project_hooks POST and PUT operations for build_events"
-merge_request: 12673
-author: Richard Clamp

changelogs/unreleased/12892-reset-css-text-align-to-initial-for-rtl.md

----
-title: "reset text-align to initial to let elements with dir="auto" align texts to right in RTL languages ( default is left )"
-merge_request: 12892
-author: goshhob

changelogs/unreleased/13711-allow-same-period-housekeeping.yml

@@ -2,5 +2,5 @@
 title: Allow to use same periods for different housekeeping tasks (effectively
   skipping the lesser task)
 merge_request: 13711
-author: @cernvcs
+author: cernvcs
 type: added

changelogs/unreleased/22619-add-an-email-address-to-unsubscribe-list-header-in-email

----
-title: Handle unsubscribe from email notifications via replying to reply+%{key}+unsubscribe@ address
-merge_request: 6597
-author:

changelogs/unreleased/26908-make-timelogs-use-foreign-keys

----
-title: Refactor Timelogs structure to use foreign keys.
-merge_request: 8769
-author:

changelogs/unreleased/32340-correct-jobs-api-documentation

----
-title: "Correction to documention for manual steps on the Jobs API"
-merge_request: 11411
-author: Zac Sturgess
\ No newline at end of file

changelogs/unreleased/35942-api-binary-encoding.yaml

----
-title: "Fix API to serve binary diffs that are treated as text."
-merge_request: 14038

changelogs/unreleased/36953-add-gitLab-pages-version-to-admin-dashboard.yml

 ---
 title: Add GitLab-Pages version to Admin Dashboard
 merge_request: 14040
-author: @travismiller
+author: travismiller
 type: added

changelogs/unreleased/add-mock-deployment-and-monitoring-service-for-development.yaml

----
-title: Added mock deployment and monitoring service with environments fixtures
-merge_request:
-author:

changelogs/unreleased/lint-changelog-yaml.yml

+---
+title: Detect when changelog entries are invalid
+merge_request:
+author:
+type: other

changelogs/unreleased/repository-name-emojis

----
-title: Added ability to put emojis into repository name
-merge_request: 7420
-author: Vincent Composieux

ee/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_code_quality.vue

@@ -111,18 +111,15 @@ export default {
 
     this.isLoading = true;
 
-    this.service.fetchCodeclimate(head_path)
-      .then(resp => resp.json())
-      .then((data) => {
-        this.mr.setCodeclimateHeadMetrics(data);
-        this.service.fetchCodeclimate(base_path)
-          .then(response => response.json())
-          .then(baseData => this.mr.setCodeclimateBaseMetrics(baseData))
-          .then(() => this.mr.compareCodeclimateMetrics())
-          .then(() => {
-            this.isLoading = false;
-          })
-          .catch(() => this.handleError());
+    Promise.all([
+      this.service.fetchCodeclimate(head_path)
+        .then(resp => resp.json()),
+      this.service.fetchCodeclimate(base_path)
+        .then(resp => resp.json()),
+    ])
+      .then((values) => {
+        this.mr.compareCodeclimateMetrics(values[0], values[1]);
+        this.isLoading = false;
       })
       .catch(() => this.handleError());
   },

ee/app/assets/javascripts/vue_merge_request_widget/stores/mr_widget_store.js

@@ -53,24 +53,12 @@ export default class MergeRequestStore extends CEMergeRequestStore {
   initCodeclimate(data) {
     this.codeclimate = data.codeclimate;
     this.codeclimateMetrics = {
-      headIssues: [],
-      baseIssues: [],
       newIssues: [],
       resolvedIssues: [],
     };
   }
 
-  setCodeclimateHeadMetrics(data) {
-    this.codeclimateMetrics.headIssues = data;
-  }
-
-  setCodeclimateBaseMetrics(data) {
-    this.codeclimateMetrics.baseIssues = data;
-  }
-
-  compareCodeclimateMetrics() {
-    const { headIssues, baseIssues } = this.codeclimateMetrics;
-
+  compareCodeclimateMetrics(headIssues, baseIssues) {
     this.codeclimateMetrics.newIssues = this.filterByFingerprint(headIssues, baseIssues);
     this.codeclimateMetrics.resolvedIssues = this.filterByFingerprint(baseIssues, headIssues);
   }

ee/lib/ee/gitlab/deltas.rb

@@ -7,6 +7,9 @@ module EE
         begin
           tree_a = repo.lookup(change[:oldrev])
           tree_b = repo.lookup(change[:newrev])
+
+          return size_of_deltas unless diffable?(tree_a) && diffable?(tree_b)
+
           diff = tree_a.diff(tree_b)
 
           diff.each_delta do |d|
@@ -25,6 +28,10 @@ module EE
           size_of_deltas
         end
       end
+
+      def self.diffable?(object)
+        [Rugged::Commit, Rugged::Tree].include?(object.class)
+      end
     end
   end
 end

lib/gitlab/checks/change_access.rb

@@ -162,14 +162,17 @@ module Gitlab
           # if newrev is blank, the branch was deleted
           return if deletion? || !(commit_validation || validate_path_locks?)
 
-          commits.each do |commit|
-            if commit_validation
-              error = check_commit(commit, push_rule)
-              raise GitAccess::UnauthorizedError, error if error
-            end
-
-            if error = check_commit_diff(commit, push_rule)
-              raise GitAccess::UnauthorizedError, error
+          # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593
+          Gitlab::GitalyClient.allow_n_plus_1_calls do
+            commits.each do |commit|
+              if commit_validation
+                error = check_commit(commit, push_rule)
+                raise GitAccess::UnauthorizedError, error if error
+              end
+
+              if error = check_commit_diff(commit, push_rule)
+                raise GitAccess::UnauthorizedError, error
+              end
             end
           end
         end

lib/gitlab/git/repository.rb

@@ -23,11 +23,8 @@ module Gitlab
       TagExistsError = Class.new(StandardError)
 
       class << self
-        # Unlike `new`, `create` takes the storage path, not the storage name
-        def create(storage_path, name, bare: true, symlink_hooks_to: nil)
-          repo_path = File.join(storage_path, name)
-          repo_path += '.git' unless repo_path.end_with?('.git')
-
+        # Unlike `new`, `create` takes the repository path
+        def create(repo_path, bare: true, symlink_hooks_to: nil)
           FileUtils.mkdir_p(repo_path, mode: 0770)
 
           # Equivalent to `git --git-path=#{repo_path} init [--bare]`

lib/gitlab/gitaly_client/repository_service.rb

@@ -53,6 +53,11 @@ module Gitlab
 
         GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
       end
+
+      def create_repository
+        request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
+        GitalyClient.call(@storage, :repository_service, :create_repository, request)
+      end
     end
   end
 end

lib/gitlab/shell.rb

@@ -65,7 +65,7 @@ module Gitlab
 
     # Init new repository
     #
-    # storage - project's storage path
+    # storage - project's storage name
     # name - project path with namespace
     #
     # Ex.
@@ -73,7 +73,19 @@ module Gitlab
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def add_repository(storage, name)
-      Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
+      relative_path = name.dup
+      relative_path << '.git' unless relative_path.end_with?('.git')
+
+      gitaly_migrate(:create_repository) do |is_enabled|
+        if is_enabled
+          repository = Gitlab::Git::Repository.new(storage, relative_path, '')
+          repository.gitaly_repository_client.create_repository
+          true
+        else
+          repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path)
+          Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path)
+        end
+      end
     rescue => err
       Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}")
       false

lib/tasks/gitlab/shell.rake

@@ -79,7 +79,7 @@ namespace :gitlab do
         if File.exist?(path_to_repo)
           print '-'
         else
-          if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+          if Gitlab::Shell.new.add_repository(project.repository_storage,
                                               project.disk_path)
             print '.'
           else

qa/qa/page/admin/menu.rb

@@ -3,15 +3,10 @@ module QA
     module Admin
       class Menu < Page::Base
         def go_to_license
-          within_middle_menu { click_link 'License' }
-        end
-
-        private
-
-        def within_middle_menu
-          page.within('.nav-control') do
-            yield
-          end
+          link = find_link 'License'
+          # Click space to scroll this link into the view
+          link.send_keys(:space)
+          link.click
         end
       end
     end

scripts/lint-changelog-yaml

+#!/usr/bin/env ruby
+
+require 'yaml'
+
+invalid_changelogs = Dir['changelogs/**/*'].reject do |changelog|
+  next true if changelog =~ /(archive\.md|unreleased(-ee)?)$/
+  next false unless changelog.end_with?('.yml')
+
+  begin
+    YAML.load_file(changelog)
+  rescue
+  end
+end
+
+if invalid_changelogs.any?
+  puts "Invalid changelogs found!\n"
+  puts invalid_changelogs.sort
+  exit 1
+else
+  puts "All changelogs are valid YAML.\n"
+  exit 0
+end

scripts/static-analysis

@@ -13,7 +13,8 @@ tasks = [
   %w[yarn run eslint],
   %w[bundle exec rubocop --require rubocop-rspec],
   %w[scripts/lint-conflicts.sh],
-  %w[bundle exec rake gettext:lint]
+  %w[bundle exec rake gettext:lint],
+  %w[scripts/lint-changelog-yaml]
 ]
 
 failed_tasks = tasks.reduce({}) do |failures, task|

spec/ee/spec/lib/ee/gitlab/deltas_spec.rb

@@ -12,5 +12,19 @@ describe EE::Gitlab::Deltas do
 
       expect(described_class.delta_size_check(change, project.repository)).to be > 0
     end
+
+    it 'handles annotated tags on an object' do
+      object_id = 'faaf198af3a36dbf41961466703cc1d47c61d051'
+      options = { message: 'test tag message\n',
+                  tagger: { name: 'John Smith', email: 'john@gmail.com' } }
+      result = project.repository.rugged.tags.create('annotated-tag', object_id, options)
+
+      change = {
+        oldrev: result.annotation.oid,
+        newrev: TestEnv::BRANCH_SHA['master']
+      }
+
+      expect(described_class.delta_size_check(change, project.repository)).to eq(0)
+    end
   end
 end

spec/javascripts/vue_mr_widget/stores/mr_widget_store_spec.js

@@ -54,36 +54,9 @@ describe('MergeRequestStore', () => {
     });
   });
 
-  describe('setCodeclimateHeadMetrics', () => {
-    it('should set defaults', () => {
-      expect(store.codeclimate).toEqual(mockData.codeclimate);
-      expect(store.codeclimateMetrics).toEqual({
-        headIssues: [],
-        baseIssues: [],
-        newIssues: [],
-        resolvedIssues: [],
-      });
-    });
-
-    it('should set the provided head metrics', () => {
-      store.setCodeclimateHeadMetrics(headIssues);
-      expect(store.codeclimateMetrics.headIssues).toEqual(headIssues);
-    });
-  });
-
-  describe('setCodeclimateBaseMetrics', () => {
-    it('should set the provided base metrics', () => {
-      store.setCodeclimateBaseMetrics(baseIssues);
-
-      expect(store.codeclimateMetrics.baseIssues).toEqual(baseIssues);
-    });
-  });
-
   describe('compareCodeclimateMetrics', () => {
     beforeEach(() => {
-      store.setCodeclimateHeadMetrics(headIssues);
-      store.setCodeclimateBaseMetrics(baseIssues);
-      store.compareCodeclimateMetrics();
+      store.compareCodeclimateMetrics(headIssues, baseIssues);
     });
 
     it('should return the new issues', () => {

spec/lib/gitlab/shell_spec.rb

@@ -451,30 +451,42 @@ describe Gitlab::Shell do
     end
 
     describe '#add_repository' do
-      it 'creates a repository' do
-        created_path = File.join(TestEnv.repos_path, 'project', 'path.git')
-        hooks_path = File.join(created_path, 'hooks')
+      shared_examples '#add_repository' do
+        let(:repository_storage) { 'default' }
+        let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
+        let(:repo_name) { 'project/path' }
+        let(:created_path) { File.join(repository_storage_path, repo_name + '.git') }
 
-        begin
-          result = gitlab_shell.add_repository(TestEnv.repos_path, 'project/path')
-
-          repo_stat = File.stat(created_path) rescue nil
-          hooks_stat = File.lstat(hooks_path) rescue nil
-          hooks_dir = File.realpath(hooks_path)
-        ensure
+        after do
           FileUtils.rm_rf(created_path)
         end
 
-        expect(result).to be_truthy
-        expect(repo_stat.mode & 0o777).to eq(0o770)
-        expect(hooks_stat.symlink?).to be_truthy
-        expect(hooks_dir).to eq(gitlab_shell_hooks_path)
+        it 'creates a repository' do
+          expect(gitlab_shell.add_repository(repository_storage, repo_name)).to be_truthy
+
+          expect(File.stat(created_path).mode & 0o777).to eq(0o770)
+
+          hooks_path = File.join(created_path, 'hooks')
+          expect(File.lstat(hooks_path)).to be_symlink
+          expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path)
+        end
+
+        it 'returns false when the command fails' do
+          FileUtils.mkdir_p(File.dirname(created_path))
+          # This file will block the creation of the repo's .git directory. That
+          # should cause #add_repository to fail.
+          FileUtils.touch(created_path)
+
+          expect(gitlab_shell.add_repository(repository_storage, repo_name)).to be_falsy
+        end
       end
 
-      it 'returns false when the command fails' do
-        expect(FileUtils).to receive(:mkdir_p).and_raise(Errno::EEXIST)
+      context 'with gitlay' do
+        it_behaves_like '#add_repository'
+      end
 
-        expect(gitlab_shell.add_repository('current/storage', 'project/path')).to be_falsy
+      context 'without gitaly', skip_gitaly_mock: true do
+        it_behaves_like '#add_repository'
       end
     end
 

spec/lib/gitlab/workhorse_spec.rb

@@ -214,15 +214,12 @@ describe Gitlab::Workhorse do
       end
 
       it 'includes a Repository param' do
-        repo_param = { Repository: {
+        repo_param = {
           storage_name: 'default',
-          relative_path: project.full_path + '.git',
-          git_object_directory: '',
-          git_alternate_object_directories: [],
-          gl_repository: ''
-        } }
+          relative_path: project.full_path + '.git'
+        }
 
-        expect(subject).to include(repo_param)
+        expect(subject[:Repository]).to include(repo_param)
       end
 
       context "when git_upload_pack action is passed" do

spec/models/project_spec.rb

@@ -1484,7 +1484,7 @@ describe Project do
     context 'using a regular repository' do
       it 'creates the repository' do
         expect(shell).to receive(:add_repository)
-          .with(project.repository_storage_path, project.disk_path)
+          .with(project.repository_storage, project.disk_path)
           .and_return(true)
 
         expect(project.repository).to receive(:after_create)
@@ -1494,7 +1494,7 @@ describe Project do
 
       it 'adds an error if the repository could not be created' do
         expect(shell).to receive(:add_repository)
-          .with(project.repository_storage_path, project.disk_path)
+          .with(project.repository_storage, project.disk_path)
           .and_return(false)
 
         expect(project.repository).not_to receive(:after_create)
@@ -1551,7 +1551,7 @@ describe Project do
         .and_return(false)
 
       expect(shell).to receive(:add_repository)
-        .with(project.repository_storage_path, project.disk_path)
+        .with(project.repository_storage, project.disk_path)
         .and_return(true)
 
       project.ensure_repository

spec/services/projects/create_service_spec.rb

@@ -156,10 +156,11 @@ describe Projects::CreateService, '#execute' do
         }
       end
 
-      let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+      let(:repository_storage) { 'default' }
+      let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
       before do
-        gitlab_shell.add_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+        gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
       end
 
       after do

spec/services/projects/fork_service_spec.rb

@@ -76,10 +76,11 @@ describe Projects::ForkService do
     end
 
     context 'repository already exists' do
-      let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+      let(:repository_storage) { 'default' }
+      let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
       before do
-        gitlab_shell.add_repository(repository_storage_path, "#{@to_user.namespace.full_path}/#{@from_project.path}")
+        gitlab_shell.add_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}")
       end
 
       after do

spec/services/projects/transfer_service_spec.rb

@@ -121,11 +121,14 @@ describe Projects::TransferService do
   end
 
   context 'namespace which contains orphan repository with same projects path name' do
-    let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+    let(:repository_storage) { 'default' }
+    let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
     before do
       group.add_owner(user)
-      gitlab_shell.add_repository(repository_storage_path, "#{group.full_path}/#{project.path}")
+      unless gitlab_shell.add_repository(repository_storage, "#{group.full_path}/#{project.path}")
+        raise 'failed to add repository'
+      end
 
       @result = transfer_project(project, user, group)
     end

spec/services/projects/update_service_spec.rb

@@ -150,10 +150,11 @@ describe Projects::UpdateService, '#execute' do
   end
 
   context 'when renaming a project' do
-    let(:repository_storage_path) { Gitlab.config.repositories.storages['default']['path'] }
+    let(:repository_storage) { 'default' }
+    let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] }
 
     before do
-      gitlab_shell.add_repository(repository_storage_path, "#{user.namespace.full_path}/existing")
+      gitlab_shell.add_repository(repository_storage, "#{user.namespace.full_path}/existing")
     end
 
     after do