Merge remote-tracking branch 'upstream/master' into ce-to-ee-2018-03-30

# Conflicts:
#	doc/api/jobs.md
#	locale/gitlab.pot

[ci skip]

.gitlab-ci.yml

@@ -322,8 +322,17 @@ package-and-qa:
   stage: build
   cache: {}
   when: manual
+  variables:
+    GIT_STRATEGY: none
+  before_script:
+    # We need to download the script rather than clone the repo since the
+    # package-and-qa job will not be able to run when the branch gets
+    # deleted (when merging the MR).
+    - apk add --update openssl
+    - wget https://gitlab.com/gitlab-org/gitlab-ce/raw/$CI_COMMIT_SHA/scripts/trigger-build-omnibus
+    - chmod 755 trigger-build-omnibus
   script:
-    - scripts/trigger-build-omnibus
+    - ./trigger-build-omnibus
   only:
     - //@gitlab-org/gitlab-ce
     - //@gitlab-org/gitlab-ee

app/services/projects/update_pages_service.rb

@@ -178,6 +178,9 @@ module Projects
 
     def latest_sha
       project.commit(build.ref).try(:sha).to_s
+    ensure
+      # Close any file descriptors that were opened and free libgit2 buffers
+      project.cleanup
     end
 
     def sha

changelogs/unreleased/44774-migrate-upload-task-fails-for-upload-with-store-nil.yml

+---
+title: Fixed gitlab:uploads:migrate task ignoring some uploads.
+merge_request: 18082
+author:
+type: fixed

changelogs/unreleased/sh-cleanup-pages-worker.yml

+---
+title: Free open file descriptors and libgit2 buffers in UpdatePagesService
+merge_request:
+author:
+type: performance

doc/api/jobs.md

@@ -296,8 +296,11 @@ Example of response
 
 > **Notes**:
 - [Introduced][ce-2893] in GitLab 8.5.
+<<<<<<< HEAD
 - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346]
   in [GitLab Premium][ee] 9.5.
+=======
+>>>>>>> upstream/master
 
 Get job artifacts of a project.
 
@@ -313,6 +316,7 @@ GET /projects/:id/jobs/:job_id/artifacts
 
 Example requests:
 
+<<<<<<< HEAD
 - Using the `PRIVATE-TOKEN` header:
 
     ```
@@ -330,6 +334,11 @@ Example requests:
     ```
     curl --location --header --form "job-token=$CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/1/jobs/8/artifacts"
     ```
+=======
+```
+curl --location --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/jobs/8/artifacts"
+```
+>>>>>>> upstream/master
 
 Response:
 
@@ -344,8 +353,11 @@ Response:
 
 > **Notes**:
 - [Introduced][ce-5347] in GitLab 8.10.
+<<<<<<< HEAD
 - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346]
   in [GitLab Premium][ee] 9.5.
+=======
+>>>>>>> upstream/master
 
 Download the artifacts archive from the given reference name and job provided the
 job finished successfully.

doc/ci/runners/README.md

@@ -35,7 +35,7 @@ are:
 
 A Runner that is specific only runs for the specified project(s). A shared Runner
 can run jobs for every project that has enabled the option **Allow shared Runners**
-under **Settings ➔ CI/CD**.
+under **Settings > CI/CD**.
 
 Projects with high demand of CI activity can also benefit from using specific
 Runners. By having dedicated Runners you are guaranteed that the Runner is not
@@ -76,7 +76,7 @@ Registering a specific can be done in two ways:
 To create a specific Runner without having admin rights to the GitLab instance,
 visit the project you want to make the Runner work for in GitLab:
 
-1. Go to **Settings ➔ CI/CD** to obtain the token
+1. Go to **Settings > CI/CD** to obtain the token
 1. [Register the Runner][register]
 
 ### Making an existing shared Runner specific
@@ -85,7 +85,7 @@ If you are an admin on your GitLab instance, you can turn any shared Runner into
 a specific one, but not the other way around. Keep in mind that this is a one
 way transition.
 
-1. Go to the Runners in the admin area **Overview ➔ Runners** (`/admin/runners`)
+1. Go to the Runners in the admin area **Overview > Runners** (`/admin/runners`)
    and find your Runner
 1. Enable any projects under **Restrict projects for this Runner** to be used
    with the Runner
@@ -101,7 +101,7 @@ can be changed afterwards under each Runner's settings.
 
 To lock/unlock a Runner:
 
-1. Visit your project's **Settings ➔ CI/CD**
+1. Visit your project's **Settings > CI/CD**
 1. Find the Runner you wish to lock/unlock and make sure it's enabled
 1. Click the pencil button
 1. Check the **Lock to current projects** option
@@ -115,7 +115,7 @@ you can enable the Runner also on any other project where you have Master permis
 
 To enable/disable a Runner in your project:
 
-1. Visit your project's **Settings ➔ CI/CD**
+1. Visit your project's **Settings > CI/CD**
 1. Find the Runner you wish to enable/disable
 1. Click **Enable for this project** or **Disable for this project**
 
@@ -124,6 +124,13 @@ Consider that if you don't lock your specific Runner to a specific project, any
 user with Master role in you project can assign your runner to another arbitrary
 project without requiring your authorization, so use it with caution.
 
+An admin can enable/disable a specific Runner for projects:
+
+1. Navigate to **Admin > Runners**
+2. Find the Runner you wish to enable/disable
+3. Click edit on the Runner
+4. Click **Enable** or **Disable** on the project
+
 ## Protected Runners
 
 >
@@ -136,7 +143,7 @@ Whenever a Runner is protected, the Runner picks only jobs created on
 
 To protect/unprotect Runners:
 
-1. Visit your project's **Settings ➔ CI/CD**
+1. Visit your project's **Settings > CI/CD**
 1. Find a Runner you want to protect/unprotect and make sure it's enabled
 1. Click the pencil button besides the Runner name
 1. Check the **Protected** option

lib/backup/artifacts.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('artifacts', JobArtifactUploader.root)
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir, 0700)
-    end
   end
 end

lib/backup/builds.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('builds', Settings.gitlab_ci.builds_path)
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir, 0700)
-    end
   end
 end

lib/backup/files.rb

 require 'open3'
+require_relative 'helper'
 
 module Backup
   class Files
+    include Backup::Helper
+
     attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
 
     def initialize(name, app_files_dir)
@@ -35,15 +38,22 @@ module Backup
 
     def restore
       backup_existing_files_dir
-      create_files_dir
 
-      run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
     end
 
     def backup_existing_files_dir
-      timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}")
+      timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}")
       if File.exist?(app_files_dir)
-        FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path))
+        # Move all files in the existing repos directory except . and .. to
+        # repositories.old.<timestamp> directory
+        FileUtils.mkdir_p(timestamped_files_path, mode: 0700)
+        files = Dir.glob(File.join(app_files_dir, "*"), File::FNM_DOTMATCH) - [File.join(app_files_dir, "."), File.join(app_files_dir, "..")]
+        begin
+          FileUtils.mv(files, timestamped_files_path)
+        rescue Errno::EACCES
+          access_denied_error(app_files_dir)
+        end
       end
     end
 

lib/backup/helper.rb

+module Backup
+  module Helper
+    def access_denied_error(path)
+      message = <<~EOS
+
+      ### NOTICE ###
+      As part of restore, the task tried to move existing content from #{path}.
+      However, it seems that directory contains files/folders that are not owned
+      by the user #{Gitlab.config.gitlab.user}. To proceed, please move the files
+      or folders inside #{path} to a secure location so that #{path} is empty and
+      run restore task again.
+
+      EOS
+      raise message
+    end
+  end
+end

lib/backup/lfs.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('lfs', Settings.lfs.storage_path)
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir, 0700)
-    end
   end
 end

lib/backup/pages.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('pages', Gitlab.config.pages.path)
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir, 0700)
-    end
   end
 end

lib/backup/registry.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('registry', Settings.registry.path)
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir, 0700)
-    end
   end
 end

lib/backup/repository.rb

 require 'yaml'
+require_relative 'helper'
 
 module Backup
   class Repository
+    include Backup::Helper
     # rubocop:disable Metrics/AbcSize
+
     def dump
       prepare
 
@@ -63,18 +66,27 @@ module Backup
       end
     end
 
-    def restore
+    def prepare_directories
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         path = repository_storage.legacy_disk_path
         next unless File.exist?(path)
 
-        # Move repos dir to 'repositories.old' dir
-        bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
-        FileUtils.mv(path, bk_repos_path)
-        # This is expected from gitlab:check
-        FileUtils.mkdir_p(path, mode: 02770)
+        # Move all files in the existing repos directory except . and .. to
+        # repositories.old.<timestamp> directory
+        bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
+        FileUtils.mkdir_p(bk_repos_path, mode: 0700)
+        files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
+
+        begin
+          FileUtils.mv(files, bk_repos_path)
+        rescue Errno::EACCES
+          access_denied_error(path)
+        end
       end
+    end
 
+    def restore
+      prepare_directories
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
         path_to_project_repo = path_to_repo(project)

lib/backup/uploads.rb

@@ -5,9 +5,5 @@ module Backup
     def initialize
       super('uploads', Rails.root.join('public/uploads'))
     end
-
-    def create_files_dir
-      Dir.mkdir(app_files_dir)
-    end
   end
 end

lib/tasks/gitlab/uploads/migrate.rake

@@ -25,8 +25,8 @@ namespace :gitlab do
       Upload.class_eval { include EachBatch } unless Upload < EachBatch
 
       Upload
-        .where.not(store: @to_store)
-        .where(uploader: @uploader_class.to_s,
+        .where(store: [nil, ObjectStorage::Store::LOCAL],
+               uploader: @uploader_class.to_s,
                model_type: @model_class.base_class.sti_name)
     end
   end

locale/gitlab.pot

@@ -1415,12 +1415,15 @@ msgid "Create empty repository"
 msgstr ""
 
 msgid "Create file"
+<<<<<<< HEAD
 msgstr ""
 
 msgid "Create epic"
 msgstr ""
 
 msgid "Create file"
+=======
+>>>>>>> upstream/master
 msgstr ""
 
 msgid "Create group label"
@@ -1468,9 +1471,12 @@ msgstr ""
 msgid "Creates a new branch from %{branchName} and re-directs to create a new merge request"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Creating epic"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Cron Timezone"
 msgstr ""
 
@@ -1554,6 +1560,7 @@ msgstr ""
 msgid "Directory name"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Disable"
 msgstr ""
 
@@ -1563,6 +1570,11 @@ msgstr ""
 msgid "Discover GitLab Geo."
 msgstr ""
 
+=======
+msgid "Discard draft"
+msgstr ""
+
+>>>>>>> upstream/master
 msgid "Dismiss Cycle Analytics introduction box"
 msgstr ""
 
@@ -1680,6 +1692,7 @@ msgstr ""
 msgid "Environments|You don't have any environments right now."
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Epic will be removed! Are you sure?"
 msgstr ""
 
@@ -1692,15 +1705,20 @@ msgstr ""
 msgid "Epics let you manage your portfolio of projects more efficiently and with less effort"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Error checking branch data. Please try again."
 msgstr ""
 
 msgid "Error committing changes. Please try again."
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Error creating epic"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Error fetching contributors data."
 msgstr ""
 
@@ -1946,9 +1964,12 @@ msgstr ""
 msgid "GeoNodes|Replication slot WAL:"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "GeoNodes|Replication slots:"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "GroupsTree|Create a project in this group."
 msgstr ""
 
@@ -2000,6 +2021,9 @@ msgstr ""
 msgid "Help"
 msgstr ""
 
+msgid "Help"
+msgstr ""
+
 msgid "Hide value"
 msgid_plural "Hide values"
 msgstr[0] ""
@@ -2443,9 +2467,12 @@ msgstr ""
 msgid "Login"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Make everyone on your team more productive regardless of their location. GitLab Geo creates read-only mirrors of your GitLab instance so you can reduce the time it takes to clone and fetch large repos."
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Manage all notifications"
 msgstr ""
 
@@ -3419,12 +3446,15 @@ msgstr ""
 msgid "Reviewing"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Roadmap"
 msgstr ""
 
 msgid "Run CI/CD pipelines for external repositories"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Runners"
 msgstr ""
 
@@ -3592,21 +3622,27 @@ msgstr ""
 msgid "Something went wrong when toggling the button"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Something went wrong while fetching Dependency Scanning."
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Something went wrong while fetching the projects."
 msgstr ""
 
 msgid "Something went wrong while fetching SAST."
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Something went wrong while fetching the projects."
 msgstr ""
 
 msgid "Something went wrong while fetching the registry list."
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Something went wrong. Please try again."
 msgstr ""
 
@@ -4318,9 +4354,12 @@ msgstr ""
 msgid "Upvotes"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Use Service Desk to connect with your users (e.g. to offer customer support) through email right inside GitLab"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Use the following registration token during setup:"
 msgstr ""
 
@@ -4333,9 +4372,12 @@ msgstr ""
 msgid "View and edit lines"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "View epics list"
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "View file @ "
 msgstr ""
 
@@ -4381,6 +4423,7 @@ msgstr ""
 msgid "Web IDE"
 msgstr ""
 
+<<<<<<< HEAD
 msgid "Webhooks allow you to trigger a URL if, for example, new code is pushed or a new issue is created. You can configure webhooks to listen for specific events like pushes, issues or merge requests. Group webhooks will apply to all projects in a group, allowing you to standardize webhook functionality across your entire group."
 msgstr ""
 
@@ -4390,6 +4433,8 @@ msgstr ""
 msgid "When leaving the URL blank, classification labels can still be specified whitout disabling cross project features or performing external authorization checks."
 msgstr ""
 
+=======
+>>>>>>> upstream/master
 msgid "Wiki"
 msgstr ""
 

spec/lib/backup/files_spec.rb

+require 'spec_helper'
+
+describe Backup::Files do
+  let(:progress) { StringIO.new }
+  let!(:project) { create(:project) }
+
+  before do
+    allow(progress).to receive(:puts)
+    allow(progress).to receive(:print)
+    allow(FileUtils).to receive(:mkdir_p).and_return(true)
+    allow(FileUtils).to receive(:mv).and_return(true)
+    allow(File).to receive(:exist?).and_return(true)
+    allow(File).to receive(:realpath).with("/var/gitlab-registry").and_return("/var/gitlab-registry")
+    allow(File).to receive(:realpath).with("/var/gitlab-registry/..").and_return("/var")
+
+    allow_any_instance_of(String).to receive(:color) do |string, _color|
+      string
+    end
+
+    allow_any_instance_of(described_class).to receive(:progress).and_return(progress)
+  end
+
+  describe '#restore' do
+    subject { described_class.new('registry', '/var/gitlab-registry') }
+    let(:timestamp) { Time.utc(2017, 3, 22) }
+
+    around do |example|
+      Timecop.freeze(timestamp) { example.run }
+    end
+
+    describe 'folders with permission' do
+      before do
+        allow(subject).to receive(:run_pipeline!).and_return(true)
+        allow(subject).to receive(:backup_existing_files).and_return(true)
+        allow(Dir).to receive(:glob).with("/var/gitlab-registry/*", File::FNM_DOTMATCH).and_return(["/var/gitlab-registry/.", "/var/gitlab-registry/..", "/var/gitlab-registry/sample1"])
+      end
+
+      it 'moves all necessary files' do
+        allow(subject).to receive(:backup_existing_files).and_call_original
+        expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}"))
+        subject.restore
+      end
+
+      it 'raises no errors' do
+        expect { subject.restore }.not_to raise_error
+      end
+
+      it 'calls tar command with unlink' do
+        expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
+        subject.restore
+      end
+    end
+
+    describe 'folders without permissions' do
+      before do
+        allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
+        allow(subject).to receive(:run_pipeline!).and_return(true)
+      end
+
+      it 'shows error message' do
+        expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry")
+        subject.restore
+      end
+    end
+  end
+end

spec/lib/backup/repository_spec.rb

@@ -7,6 +7,8 @@ describe Backup::Repository do
   before do
     allow(progress).to receive(:puts)
     allow(progress).to receive(:print)
+    allow(FileUtils).to receive(:mkdir_p).and_return(true)
+    allow(FileUtils).to receive(:mv).and_return(true)
 
     allow_any_instance_of(String).to receive(:color) do |string, _color|
       string
@@ -68,6 +70,17 @@ describe Backup::Repository do
         end
       end
     end
+
+    describe 'folders without permissions' do
+      before do
+        allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
+      end
+
+      it 'shows error message' do
+        expect(subject).to receive(:access_denied_error)
+        subject.restore
+      end
+    end
   end
 
   describe '#empty_repo?' do

spec/tasks/gitlab/uploads/migrate_rake_spec.rb

@@ -20,9 +20,23 @@ describe 'gitlab:uploads:migrate rake tasks' do
     run_rake_task("gitlab:uploads:migrate", *args)
   end
 
-  it 'enqueue jobs in batch' do
-    expect(ObjectStorage::MigrateUploadsWorker).to receive(:enqueue!).exactly(4).times
+  shared_examples 'enqueue jobs in batch' do |batch:|
+    it do
+      expect(ObjectStorage::MigrateUploadsWorker)
+        .to receive(:perform_async).exactly(batch).times
+              .and_return("A fake job.")
 
-    run
+      run
+    end
+  end
+
+  it_behaves_like 'enqueue jobs in batch', batch: 4
+
+  context 'Upload has store = nil' do
+    before do
+      Upload.where(model: projects).update_all(store: nil)
+    end
+
+    it_behaves_like 'enqueue jobs in batch', batch: 4
   end
 end