Merge branch 'backup_skip_option' into 'master'

Ability to skip some items in backup

https://dev.gitlab.org/gitlab/gitlabhq/issues/1936

Now it is possible to run

```
sudo gitlab-rake gitlab:backup:create SKIP=repositories,uploads
```

available options: repositories,uploads,db

See merge request !470

CHANGELOG

@@ -52,6 +52,7 @@ v 7.10.0 (unreleased)
   - Don't show commit comment button when user is not signed in.
   - Fix admin user projects lists.
   - Don't leak private group existence by redirecting from namespace controller to group controller.
+  - Ability to skip some items from backup (database, respositories or uploads)
 
 v 7.9.2
   - Contains no changes

doc/raketasks/backup_restore.md

@@ -17,6 +17,13 @@ sudo gitlab-rake gitlab:backup:create
 sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
 ```
 
+Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, 
+uploads (attachments), repositories. Use a comma to specify several options at the same time.
+
+```
+sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
+```
+
 Example output:
 
 ```

lib/backup/manager.rb

 module Backup
   class Manager
-    BACKUP_CONTENTS = %w{repositories/ db/ uploads/ backup_information.yml}
-
     def pack
       # saving additional informations
       s = {}
@@ -9,6 +7,7 @@ module Backup
       s[:backup_created_at]  = Time.now
       s[:gitlab_version]     = Gitlab::VERSION
       s[:tar_version]        = tar_version
+      s[:skipped]            = ENV["SKIP"]
       tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar"
 
       Dir.chdir(Gitlab.config.backup.path) do
@@ -17,12 +16,12 @@ module Backup
           file << s.to_yaml.gsub(/^---\n/,'')
         end
 
-        FileUtils.chmod(0700, %w{db uploads repositories})
+        FileUtils.chmod(0700, folders_to_backup)
 
         # create archive
         $progress.print "Creating backup archive: #{tar_file} ... "
         orig_umask = File.umask(0077)
-        if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS)
+        if Kernel.system('tar', '-cf', tar_file, *backup_contents)
           $progress.puts "done".green
         else
           puts "creating archive #{tar_file} failed".red
@@ -46,6 +45,7 @@ module Backup
 
       connection = ::Fog::Storage.new(connection_settings)
       directory = connection.directories.get(remote_directory)
+
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
         $progress.puts "done".green
       else
@@ -56,7 +56,10 @@ module Backup
 
     def cleanup
       $progress.print "Deleting tmp directories ... "
-      BACKUP_CONTENTS.each do |dir|
+      
+      backup_contents.each do |dir|
+        next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
         if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
           $progress.puts "done".green
         else
@@ -73,6 +76,7 @@ module Backup
 
       if keep_time > 0
         removed = 0
+        
         Dir.chdir(Gitlab.config.backup.path) do
           file_list = Dir.glob('*_gitlab_backup.tar')
           file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
@@ -84,6 +88,7 @@ module Backup
             end
           end
         end
+
         $progress.puts "done. (#{removed} removed)".green
       else
         $progress.puts "skipping".yellow
@@ -96,6 +101,7 @@ module Backup
       # check for existing backups in the backup dir
       file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i }
       puts "no backups found" if file_list.count == 0
+
       if file_list.count > 1 && ENV["BACKUP"].nil?
         puts "Found more than one backup, please specify which one you want to restore:"
         puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
@@ -110,6 +116,7 @@ module Backup
       end
 
       $progress.print "Unpacking backup ... "
+
       unless Kernel.system(*%W(tar -xf #{tar_file}))
         puts "unpacking backup failed".red
         exit 1
@@ -117,7 +124,6 @@ module Backup
         $progress.puts "done".green
       end
 
-      settings = YAML.load_file("backup_information.yml")
       ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
 
       # restoring mismatching backups can lead to unexpected problems
@@ -136,5 +142,29 @@ module Backup
       tar_version, _ = Gitlab::Popen.popen(%W(tar --version))
       tar_version.force_encoding('locale').split("\n").first
     end
+
+    def skipped?(item)
+      settings[:skipped] && settings[:skipped].include?(item)
+    end
+
+    private
+
+    def backup_contents
+      folders_to_backup + ["backup_information.yml"]
+    end
+
+    def folders_to_backup
+      folders = %w{repositories db uploads}
+
+      if ENV["SKIP"]
+        return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
+      end
+
+      folders
+    end
+
+    def settings
+      @settings ||= YAML.load_file("backup_information.yml")
+    end
   end
 end

lib/tasks/gitlab/backup.rake

@@ -27,9 +27,9 @@ namespace :gitlab do
       backup = Backup::Manager.new
       backup.unpack
 
-      Rake::Task["gitlab:backup:db:restore"].invoke
-      Rake::Task["gitlab:backup:repo:restore"].invoke
-      Rake::Task["gitlab:backup:uploads:restore"].invoke
+      Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
+      Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
+      Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -38,9 +38,14 @@ namespace :gitlab do
     namespace :repo do
       task create: :environment do
         $progress.puts "Dumping repositories ...".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
+          $progress.puts "[SKIPPED]".cyan
+        else
           Backup::Repository.new.dump
           $progress.puts "done".green
         end
+      end
 
       task restore: :environment do
         $progress.puts "Restoring repositories ...".blue
@@ -52,9 +57,14 @@ namespace :gitlab do
     namespace :db do
       task create: :environment do
         $progress.puts "Dumping database ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("db")
+          $progress.puts "[SKIPPED]".cyan
+        else
           Backup::Database.new.dump
           $progress.puts "done".green
         end
+      end
 
       task restore: :environment do
         $progress.puts "Restoring database ... ".blue
@@ -66,9 +76,14 @@ namespace :gitlab do
     namespace :uploads do
       task create: :environment do
         $progress.puts "Dumping uploads ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
+          $progress.puts "[SKIPPED]".cyan
+        else
           Backup::Uploads.new.dump
           $progress.puts "done".green
         end
+      end
 
       task restore: :environment do
         $progress.puts "Restoring uploads ... ".blue

spec/tasks/gitlab/backup_rake_spec.rb

@@ -98,4 +98,55 @@ describe 'gitlab:app namespace rake task' do
       expect(temp_dirs).to be_empty
     end
   end # backup_create task
+
+  describe "Skipping items" do
+    def tars_glob
+      Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
+    end
+
+    before :all do
+      @origin_cd = Dir.pwd
+
+      Rake::Task["gitlab:backup:db:create"].reenable
+      Rake::Task["gitlab:backup:repo:create"].reenable
+      Rake::Task["gitlab:backup:uploads:create"].reenable
+
+      # Record the existing backup tars so we don't touch them
+      existing_tars = tars_glob
+
+      # Redirect STDOUT and run the rake task
+      orig_stdout = $stdout
+      $stdout = StringIO.new
+      ENV["SKIP"] = "repositories"
+      run_rake_task('gitlab:backup:create')
+      $stdout = orig_stdout
+
+      @backup_tar = (tars_glob - existing_tars).first
+    end
+
+    after :all do
+      FileUtils.rm(@backup_tar)
+      Dir.chdir @origin_cd
+    end
+
+    it "does not contain skipped item" do
+      tar_contents, exit_status = Gitlab::Popen.popen(
+        %W{tar -tvf #{@backup_tar} db uploads repositories}
+      )
+
+      expect(tar_contents).to match('db/')
+      expect(tar_contents).to match('uploads/')
+      expect(tar_contents).not_to match('repositories/')
+    end
+
+    it 'does not invoke repositories restore' do
+      Rake::Task["gitlab:shell:setup"].stub invoke: true
+      allow($stdout).to receive :write
+
+      expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
+      expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
+      expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
+      expect { run_rake_task('gitlab:backup:restore') }.to_not raise_error
+    end
+  end
 end # gitlab:app namespace