Merge branch 'ce-to-ee' into 'master'

CE Upstream - Monday

Closes omnibus-gitlab#2988

See merge request gitlab-org/gitlab-ee!3558

Gemfile

@@ -295,7 +295,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~> 0.7.0.beta36'
+  gem 'prometheus-client-mmap', '~> 0.7.0.beta37'
   gem 'raindrops', '~> 0.18'
 end
 

Gemfile.lock

@@ -654,7 +654,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.7.0.beta36)
+    prometheus-client-mmap (0.7.0.beta37)
       mmap2 (~> 2.2, >= 2.2.9)
     pry (0.10.4)
       coderay (~> 1.1.0)
@@ -1149,7 +1149,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.7.0.beta36)
+  prometheus-client-mmap (~> 0.7.0.beta37)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

changelogs/unreleased/40291-ignore-hashed-repos-cleanup-repositories.yml

+---
+title: Ensure that rake gitlab:cleanup:repos task does not mess with hashed repositories
+merge_request: 15520
+author:
+type: fixed

changelogs/unreleased/40352-ignore-hashed-repos-cleanup-dirs.yml

+---
+title: Ensure that rake gitlab:cleanup:dirs task does not mess with hashed repositories
+merge_request: 15600
+author:
+type: fixed

changelogs/unreleased/fl-upgrade-svg.yml

+---
+title: Update svg external depencency
+merge_request:
+author:
+type: other

doc/articles/index.md

@@ -26,6 +26,7 @@ Build, test, and deploy the software you develop with [GitLab CI/CD](../ci/READM
 
 | Article title | Category | Publishing date |
 | :------------ | :------: | --------------: |
+| [Autoscaling GitLab Runners on AWS](runner_autoscale_aws/index.md) | Admin guide | 2017-11-24 |
 | [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](laravel_with_gitlab_and_envoy/index.md) | Tutorial | 2017-08-31 |
 | [How to deploy Maven projects to Artifactory with GitLab CI/CD](artifactory_and_gitlab/index.md) | Tutorial | 2017-08-15 |
 | [Making CI Easier with GitLab](https://about.gitlab.com/2017/07/13/making-ci-easier-with-gitlab/) | Concepts | 2017-07-13 |

doc/user/project/pipelines/schedules.md

@@ -5,7 +5,7 @@
 - In 9.2, the feature was [renamed to Pipeline Schedule][ce-10853].
 - Cron notation is parsed by [Rufus-Scheduler](https://github.com/jmettraux/rufus-scheduler).
 
-Pipeline schedules can be used to run pipelines only once, or for example every
+Pipeline schedules can be used to run a pipeline at specific intervals, for example every
 month on the 22nd for a certain branch.
 
 ## Using Pipeline schedules

lib/tasks/gitlab/cleanup.rake

 namespace :gitlab do
   namespace :cleanup do
+    HASHED_REPOSITORY_NAME = '@hashed'.freeze
+
     desc "GitLab | Cleanup | Clean namespaces"
     task dirs: :environment  do
       warn_user_is_not_gitlab
       remove_flag = ENV['REMOVE']
 
-      namespaces = Namespace.pluck(:path)
+      namespaces  = Namespace.pluck(:path)
+      namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         git_base_path = repository_storage['path']
         all_dirs = Dir.glob(git_base_path + '/*')
@@ -91,7 +94,7 @@ namespace :gitlab do
 
             # TODO ignoring hashed repositories for now.  But revisit to fully support
             # possible orphaned hashed repos
-            next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace)
+            next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
 
             new_path = path + move_suffix
             puts path.inspect + ' -> ' + new_path.inspect

spec/tasks/gitlab/cleanup_rake_spec.rb

@@ -5,7 +5,7 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
 
-  context 'cleanup repositories' do
+  describe 'cleanup' do
     let(:gitaly_address) { Gitlab.config.repositories.storages.default.gitaly_address }
     let(:storages) do
       {
@@ -22,20 +22,46 @@ describe 'gitlab:cleanup rake tasks' do
       FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
     end
 
-    it 'moves it to an orphaned path' do
-      FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/broken/project.git'))
-      run_rake_task('gitlab:cleanup:repos')
-      repo_list = Dir['tmp/tests/default_storage/broken/*']
+    describe 'cleanup:repos' do
+      before do
+        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/broken/project.git'))
+        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))
+      end
 
-      expect(repo_list.first).to include('+orphaned+')
+      it 'moves it to an orphaned path' do
+        run_rake_task('gitlab:cleanup:repos')
+        repo_list = Dir['tmp/tests/default_storage/broken/*']
+
+        expect(repo_list.first).to include('+orphaned+')
+      end
+
+      it 'ignores @hashed repos' do
+        run_rake_task('gitlab:cleanup:repos')
+
+        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy
+      end
     end
 
-    it 'ignores @hashed repos' do
-      FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))
+    describe 'cleanup:dirs' do
+      it 'removes missing namespaces' do
+        FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_1/project.git"))
+        FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_2/project.git"))
+        allow(Namespace).to receive(:pluck).and_return('namespace_1')
+
+        stub_env('REMOVE', 'true')
+        run_rake_task('gitlab:cleanup:dirs')
+
+        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_1'))).to be_truthy
+        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_2'))).to be_falsey
+      end
+
+      it 'ignores @hashed directory' do
+        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))
 
-      run_rake_task('gitlab:cleanup:repos')
+        run_rake_task('gitlab:cleanup:dirs')
 
-      expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy
+        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy
+      end
     end
   end
 end