Merge branch 'ce-to-ee-2018-01-25' into 'master'

CE upstream - 2018-01-25 21:23 UTC

Closes gitaly#946

See merge request gitlab-org/gitlab-ee!4263

.gitlab-ci.yml

@@ -413,7 +413,6 @@ spinach-mysql 2 3: *spinach-metadata-mysql
 
 # Static analysis jobs
 .ruby-static-analysis: &ruby-static-analysis
-  <<: *pull-cache
   variables:
     SIMPLECOV: "false"
     SETUP_DB: "false"
@@ -434,6 +433,12 @@ static-analysis:
   stage: test
   script:
     - scripts/static-analysis
+  cache:
+    key: "ruby-2.3.6-with-yarn-and-rubocop"
+    paths:
+      - vendor/ruby
+      - .yarn-cache/
+      - tmp/rubocop_cache
 
 # Documentation checks:
 # - Check validity of relative links
@@ -742,8 +747,6 @@ pages:
 cache gems:
   <<: *dedicated-runner
   <<: *pull-cache
-  only:
-    - tags
   variables:
     SETUP_DB: "false"
   script:
@@ -754,6 +757,7 @@ cache gems:
   only:
     - master@gitlab-org/gitlab-ce
     - master@gitlab-org/gitlab-ee
+    - tags
 
 gitlab_git_test:
   <<: *dedicated-runner

.rubocop.yml

@@ -18,6 +18,7 @@ AllCops:
     - 'bin/**/*'
     - 'generator_templates/**/*'
     - 'builds/**/*'
+  CacheRootDirectory: tmp
 
 # This cop checks whether some constant value isn't a
 # mutable literal (e.g. array or hash).

GITALY_SERVER_VERSION

-0.73.0
+0.74.0

Gemfile

@@ -422,7 +422,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.76.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.78.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -309,7 +309,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.76.0)
+    gitaly-proto (0.78.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1091,7 +1091,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.76.0)
+  gitaly-proto (~> 0.78.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)

app/assets/javascripts/api.js

@@ -180,7 +180,7 @@ const Api = {
 
   issueTemplate(namespacePath, projectPath, key, type, callback) {
     const url = Api.buildUrl(Api.issuableTemplatePath)
-      .replace(':key', key)
+      .replace(':key', encodeURIComponent(key))
       .replace(':type', type)
       .replace(':project_path', projectPath)
       .replace(':namespace_path', namespacePath);

app/assets/javascripts/render_math.js

@@ -18,7 +18,7 @@ function renderWithKaTeX(elements) {
 
     const display = $this.attr('data-math-style') === 'display';
     try {
-      katex.render($this.text(), mathNode.get(0), { displayMode: display });
+      katex.render($this.text(), mathNode.get(0), { displayMode: display, throwOnError: false });
       mathNode.insertAfter($this);
       $this.remove();
     } catch (err) {

app/models/repository.rb

@@ -25,7 +25,7 @@ class Repository
   attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
-  delegate :bundle_to_disk, to: :raw_repository
+  delegate :bundle_to_disk, :create_from_bundle, to: :raw_repository
 
   CreateTreeError = Class.new(StandardError)
 

app/views/projects/_readme.html.haml

@@ -9,15 +9,15 @@
 
 - else
   .row-content-block.second-block.center
-    %h3.page-title
+    %h4
       This project does not have a README yet
+
     - if can?(current_user, :push_code, @project)
       %p
         A
         %code README
         file contains information about other files in a repository and is commonly
         distributed with computer software, forming part of its documentation.
+        GitLab will render it here instead of this message.
       %p
-        We recommend you to
-        = link_to "add a README", add_special_file_path(@project, file_name: 'README.md')
-        file to the repository and GitLab will render it here instead of this message.
+        = link_to "Add Readme", add_special_file_path(@project, file_name: 'README.md'),  class: 'btn btn-new'

app/views/projects/buttons/_dropdown.html.haml

@@ -6,7 +6,10 @@
     %ul.dropdown-menu.dropdown-menu-align-right.project-home-dropdown
       - can_create_issue = can?(current_user, :create_issue, @project)
       - merge_project = can?(current_user, :create_merge_request, @project) ? @project : (current_user && current_user.fork_of(@project))
-      - can_create_snippet = can?(current_user, :create_snippet, @project)
+      - can_create_project_snippet = can?(current_user, :create_project_snippet, @project)
+
+      - if can_create_issue || merge_project || can_create_project_snippet
+        %li.dropdown-header= _('This project')
 
       - if can_create_issue
         %li= link_to _('New issue'), new_project_issue_path(@project)
@@ -14,11 +17,11 @@
       - if merge_project
         %li= link_to _('New merge request'), project_new_merge_request_path(merge_project)
 
-      - if can_create_snippet
+      - if can_create_project_snippet
         %li= link_to _('New snippet'), new_project_snippet_path(@project)
 
-      - if can_create_issue || merge_project || can_create_snippet
-        %li.divider
+      - if can?(current_user, :push_code, @project)
+        %li.dropdown-header= _('This repository')
 
       - if can?(current_user, :push_code, @project)
         %li= link_to _('New file'), project_new_blob_path(@project, @project.default_branch || 'master')
@@ -31,5 +34,5 @@
         - continue_params = { to:         project_new_blob_path(@project, @project.default_branch || 'master'),
                               notice:     edit_in_new_fork_notice,
                               notice_now: edit_in_new_fork_notice_now }
-        - fork_path = project_forks_path(@project, namespace_key:  current_user.namespace.id, continue: continue_params)
+        - fork_path = project_forks_path(@project, namespace_key: current_user.namespace.id, continue: continue_params)
         %li= link_to _('New file'), fork_path, method: :post

app/views/projects/empty.html.haml

@@ -6,8 +6,9 @@
 = render "home_panel"
 
 .row-content-block.second-block.center
-  %h3.page-title
+  %h4
     The repository for this project is empty
+
   - if can?(current_user, :push_code, @project)
     %p
       If you already have files you can push them using command line instructions below.
@@ -28,8 +29,8 @@
       %p
         - link = link_to(s_('AutoDevOps|Auto DevOps (Beta)'), project_settings_ci_cd_path(@project, anchor: 'js-general-pipeline-settings'))
         = s_('AutoDevOps|You can activate %{link_to_settings} for this project.').html_safe % { link_to_settings: link }
-      %p
-        = s_('AutoDevOps|It will automatically build, test, and deploy your application based on a predefined CI/CD configuration.')
+      %p= s_('AutoDevOps|It will automatically build, test, and deploy your application based on a predefined CI/CD configuration.')
+      %p= link_to _('New file'), project_new_blob_path(@project, @project.default_branch || 'master'), class: 'btn btn-new'
 
 - if can?(current_user, :push_code, @project)
   %div{ class: container_class }
@@ -79,4 +80,4 @@
 
           - if can? current_user, :remove_project, @project
             .prepend-top-20
-              = link_to 'Remove project', [@project.namespace.becomes(Namespace), @project], data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
+              = link_to 'Remove project', [@project.namespace.becomes(Namespace), @project], data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-inverted btn-remove pull-right"

app/views/users/show.html.haml

@@ -58,15 +58,15 @@
                 = icon('skype')
           - unless @user.linkedin.blank?
             .profile-link-holder.middle-dot-divider
-              = link_to linkedin_url(@user), title: "LinkedIn" do
+              = link_to linkedin_url(@user), title: "LinkedIn", target: '_blank', rel: 'noopener noreferrer nofollow' do
                 = icon('linkedin-square')
           - unless @user.twitter.blank?
             .profile-link-holder.middle-dot-divider
-              = link_to twitter_url(@user), title: "Twitter" do
+              = link_to twitter_url(@user), title: "Twitter", target: '_blank', rel: 'noopener noreferrer nofollow' do
                 = icon('twitter-square')
           - unless @user.website_url.blank?
             .profile-link-holder.middle-dot-divider
-              = link_to @user.short_website_url, @user.full_website_url, class: 'text-link'
+              = link_to @user.short_website_url, @user.full_website_url, class: 'text-link', target: '_blank', rel: 'noopener noreferrer nofollow'
           - unless @user.location.blank?
             .profile-link-holder.middle-dot-divider
               = icon('map-marker')

changelogs/unreleased/40028-special-characters-on-issuable-templates.yml

+---
+title: Handle special characters on API request of issuable templates
+merge_request: 15323
+author: Takuya Noguchi
+type: fixed

changelogs/unreleased/disable-throwOnError-in-katex.yml

+---
+title: Disable throwOnError in KaTeX to reveal user where is the problem
+merge_request: 16684
+author: Jakub Jirutka
+type: other

changelogs/unreleased/feat-add-section-headers-to-project-repo-buttons.yml

+---
+title: Improve empty project overview
+merge_request: 16617
+author: George Tsiolis
+type: added

changelogs/unreleased/update-node-docs.yml

+---
+title: fix documentation about node version
+merge_request: 16720
+author: Tobias Gurtzick
+type: other

config/routes/project.rb

@@ -40,7 +40,7 @@ constraints(ProjectUrlConstrainer.new) do
       #
       # Templates
       #
-      get '/templates/:template_type/:key' => 'templates#show', as: :template
+      get '/templates/:template_type/:key' => 'templates#show', as: :template, constraints: { key: /[^\/]+/ }
 
       resource  :avatar, only: [:show, :destroy]
       resources :commit, only: [:show], constraints: { id: /\h{7,40}/ } do

doc/update/10.2-to-10.3.md

@@ -54,17 +54,16 @@ sudo gem install bundler --no-ri --no-rdoc
 
 ### 4. Update Node
 
-GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets and
-it has a minimum requirement of node v4.3.0.
+GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets.
+We require a minimum version of node v6.0.0.
 
 You can check which version you are running with `node -v`. If you are running
-a version older than `v4.3.0` you will need to update to a newer version.  You
+a version older than `v6.0.0` you will need to update to a newer version. You
 can find instructions to install from community maintained packages or compile
 from source at the nodejs.org website.
 
 <https://nodejs.org/en/download/>
 
-
 Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage
 JavaScript dependencies.
 

doc/update/10.3-to-10.4.md

@@ -56,17 +56,16 @@ sudo gem install bundler --no-ri --no-rdoc
 
 ### 4. Update Node
 
-GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets and
-it has a minimum requirement of node v4.3.0.
+GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets.
+We require a minimum version of node v6.0.0.
 
 You can check which version you are running with `node -v`. If you are running
-a version older than `v4.3.0` you will need to update to a newer version.  You
+a version older than `v6.0.0` you will need to update to a newer version. You
 can find instructions to install from community maintained packages or compile
 from source at the nodejs.org website.
 
 <https://nodejs.org/en/download/>
 
-
 Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage
 JavaScript dependencies.
 

lib/gitlab/git/blob.rb

@@ -70,11 +70,9 @@ module Gitlab
         # Returns array of Gitlab::Git::Blob
         # Does not guarantee blob data will be set
         def batch_lfs_pointers(repository, blob_ids)
-          return [] if blob_ids.empty?
-
           repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled|
             if is_enabled
-              repository.gitaly_blob_client.batch_lfs_pointers(blob_ids)
+              repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a)
             else
               blob_ids.lazy
                       .select { |sha| possible_lfs_blob?(repository, sha) }

lib/gitlab/git/repository.rb

@@ -1188,6 +1188,19 @@ module Gitlab
         end
       end
 
+      def create_from_bundle(bundle_path)
+        gitaly_migrate(:create_repo_from_bundle) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.create_from_bundle(bundle_path)
+          else
+            run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil)
+            self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
+          end
+        end
+
+        true
+      end
+
       def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
         gitaly_migrate(:rebase) do |is_enabled|
           if is_enabled

lib/gitlab/gitaly_client/blob_service.rb

@@ -34,6 +34,8 @@ module Gitlab
       end
 
       def batch_lfs_pointers(blob_ids)
+        return [] if blob_ids.empty?
+
         request = Gitaly::GetLFSPointersRequest.new(
           repository: @gitaly_repo,
           blob_ids: blob_ids

lib/gitlab/gitaly_client/repository_service.rb

@@ -3,6 +3,8 @@ module Gitlab
     class RepositoryService
       include Gitlab::EncodingHelper
 
+      MAX_MSG_SIZE = 128.kilobytes.freeze
+
       def initialize(repository)
         @repository = repository
         @gitaly_repo = repository.gitaly_repository
@@ -178,6 +180,29 @@ module Gitlab
           end
         end
       end
+
+      def create_from_bundle(bundle_path)
+        request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo)
+        enum = Enumerator.new do |y|
+          File.open(bundle_path, 'rb') do |f|
+            while data = f.read(MAX_MSG_SIZE)
+              request.data = data
+
+              y.yield request
+
+              request = Gitaly::CreateRepositoryFromBundleRequest.new
+            end
+          end
+        end
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_bundle,
+          enum,
+          timeout: GitalyClient.default_timeout
+        )
+      end
     end
   end
 end

lib/gitlab/import_export/command_line_util.rb

@@ -11,11 +11,6 @@ module Gitlab
         untar_with_options(archive: archive, dir: dir, options: 'zxf')
       end
 
-      def git_clone_bundle(repo_path:, bundle_path:)
-        execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path}))
-        Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
-      end
-
       def mkdir_p(path)
         FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
         FileUtils.chmod(DEFAULT_MODE, path)

lib/gitlab/import_export/repo_restorer.rb

@@ -13,7 +13,7 @@ module Gitlab
       def restore
         return true unless File.exist?(@path_to_bundle)
 
-        git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle)
+        @project.repository.create_from_bundle(@path_to_bundle)
       rescue => e
         @shared.error(e)
         false

spec/javascripts/api_spec.js

@@ -283,9 +283,9 @@ describe('Api', () => {
     it('fetches an issue template', (done) => {
       const namespace = 'some namespace';
       const project = 'some project';
-      const templateKey = 'template key';
+      const templateKey = ' template #%?.key ';
       const templateType = 'template type';
-      const expectedUrl = `${dummyUrlRoot}/${namespace}/${project}/templates/${templateType}/${templateKey}`;
+      const expectedUrl = `${dummyUrlRoot}/${namespace}/${project}/templates/${templateType}/${encodeURIComponent(templateKey)}`;
       spyOn(jQuery, 'ajax').and.callFake((request) => {
         expect(request.url).toEqual(expectedUrl);
         return sendDummyResponse();

spec/lib/gitlab/git/blob_spec.rb

@@ -268,6 +268,21 @@ describe Gitlab::Git::Blob, seed_helper: true do
         expect(blobs).to all( be_a(Gitlab::Git::Blob) )
       end
 
+      it 'accepts blob IDs as a lazy enumerator' do
+        blobs = described_class.batch_lfs_pointers(repository, [lfs_blob.id].lazy)
+
+        expect(blobs.count).to eq(1)
+        expect(blobs).to all( be_a(Gitlab::Git::Blob) )
+      end
+
+      it 'handles empty list of IDs gracefully' do
+        blobs_1 = described_class.batch_lfs_pointers(repository, [].lazy)
+        blobs_2 = described_class.batch_lfs_pointers(repository, [])
+
+        expect(blobs_1).to eq([])
+        expect(blobs_2).to eq([])
+      end
+
       it 'silently ignores tree objects' do
         blobs = described_class.batch_lfs_pointers(repository, [tree_object.oid])
 

spec/lib/gitlab/git/repository_spec.rb

@@ -1992,6 +1992,47 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#create_from_bundle' do
+    shared_examples 'creating repo from bundle' do
+      let(:bundle_path) { File.join(Dir.tmpdir, "repo-#{SecureRandom.hex}.bundle") }
+      let(:project) { create(:project) }
+      let(:imported_repo) { project.repository.raw }
+
+      before do
+        expect(repository.bundle_to_disk(bundle_path)).to be true
+      end
+
+      after do
+        FileUtils.rm_rf(bundle_path)
+      end
+
+      it 'creates a repo from a bundle file' do
+        expect(imported_repo).not_to exist
+
+        result = imported_repo.create_from_bundle(bundle_path)
+
+        expect(result).to be true
+        expect(imported_repo).to exist
+        expect { imported_repo.fsck }.not_to raise_exception
+      end
+
+      it 'creates a symlink to the global hooks dir' do
+        imported_repo.create_from_bundle(bundle_path)
+        hooks_path = File.join(imported_repo.path, 'hooks')
+
+        expect(File.readlink(hooks_path)).to eq(Gitlab.config.gitlab_shell.hooks_path)
+      end
+    end
+
+    context 'when Gitaly create_repo_from_bundle feature is enabled' do
+      it_behaves_like 'creating repo from bundle'
+    end
+
+    context 'when Gitaly create_repo_from_bundle feature is disabled', :disable_gitaly do
+      it_behaves_like 'creating repo from bundle'
+    end
+  end
+
   context 'gitlab_projects commands' do
     let(:gitlab_projects) { repository.gitlab_projects }
     let(:timeout) { Gitlab.config.gitlab_shell.git_timeout }