Merge branch 'backup_incremental' into 'master'

Unpack a previous backup for incremental backup

See merge request gitlab-org/gitlab!83566

doc/raketasks/backup_restore.md

@@ -375,6 +375,30 @@ For example, for installations from source:
 sudo -u git -H bundle exec rake gitlab:backup:create GITLAB_BACKUP_MAX_CONCURRENCY=4 GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY=1
 ```
 
+#### Incremental repository backups
+
+> Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`.
+On GitLab.com, this feature is not available.
+This feature is not ready for production use.
+
+Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup
+bundle for each repository. There must be an existing backup to create an incremental backup from and this backup will be overwritten. You can use the `BACKUP=timestamp_of_backup` option to choose which backup will be used.
+
+To create an incremental backup, run:
+
+```shell
+sudo gitlab-backup create INCREMENTAL=yes
+```
+
+Incremental backups can also be created from [an untarred backup](#skipping-tar-creation) by using `SKIP=tar`:
+
+```shell
+sudo gitlab-backup create INCREMENTAL=yes SKIP=tar
+```
+
 #### Uploading backups to a remote (cloud) storage
 
 You can let the backup script upload (using the [Fog library](http://fog.io/))
@@ -1830,22 +1854,3 @@ If you have a specific reason to change the path, it can be configured in Omnibu
 
 1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure)
    for the changes to take effect
-
-### Incremental repository backups
-
-> Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default.
-
-FLAG:
-On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`.
-On GitLab.com, this feature is not available.
-This feature is not ready for production use.
-
-Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup
-bundle for each repository. Because incremental backups require access to the previous backup, you can't use incremental
-backups with tar files.
-
-To create an incremental backup, run:
-
-```shell
-sudo gitlab-backup create SKIP=tar INCREMENTAL=yes
-```

lib/backup/manager.rb

@@ -19,7 +19,9 @@ module Backup
       @progress = progress
 
       force = ENV['force'] == 'yes'
-      incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false)
+      @incremental = Feature.feature_flags_available? &&
+        Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) &&
+        Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false)
 
       @definitions = definitions || {
         'db' => TaskDefinition.new(
@@ -30,8 +32,7 @@ module Backup
         'repositories' => TaskDefinition.new(
           destination_path: 'repositories',
           destination_optional: true,
-          task: Repositories.new(progress,
-                                 strategy: repository_backup_strategy(incremental))
+          task: Repositories.new(progress, strategy: repository_backup_strategy)
         ),
         'uploads' => TaskDefinition.new(
           destination_path: 'uploads.tar.gz',
@@ -69,6 +70,12 @@ module Backup
     end
 
     def create
+      if incremental?
+        unpack
+        read_backup_information
+        verify_backup_version
+      end
+
       @definitions.keys.each do |task_name|
         run_create_task(task_name)
       end
@@ -87,7 +94,7 @@ module Backup
       puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need these files to restore a backup.\n" \
            "Please back them up manually.".color(:red)
-      puts_time "Backup task is done."
+      puts_time "Backup #{backup_id} is done."
     end
 
     def run_create_task(task_name)
@@ -169,6 +176,10 @@ module Backup
 
     private
 
+    def incremental?
+      @incremental
+    end
+
     def read_backup_information
       @backup_information ||= YAML.load_file(File.join(backup_path, MANIFEST_NAME))
     end
@@ -338,8 +349,15 @@ module Backup
           puts_time 'Found more than one backup:'
           # print list of available backups
           puts_time " " + available_timestamps.join("\n ")
-          puts_time 'Please specify which one you want to restore:'
-          puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
+
+          if incremental?
+            puts_time 'Please specify which one you want to create an incremental backup for:'
+            puts_time 'rake gitlab:backup:create INCREMENTAL=true BACKUP=timestamp_of_backup'
+          else
+            puts_time 'Please specify which one you want to restore:'
+            puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
+          end
+
           exit 1
         end
 
@@ -437,11 +455,15 @@ module Backup
     end
 
     def tar_file
-      @tar_file ||= if ENV['BACKUP'].present?
-                      File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX
-                    else
-                      "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}"
-                    end
+      @tar_file ||= "#{backup_id}#{FILE_NAME_SUFFIX}"
+    end
+
+    def backup_id
+      @backup_id ||= if ENV['BACKUP'].present?
+                       File.basename(ENV['BACKUP'])
+                     else
+                       "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}"
+                     end
     end
 
     def create_attributes
@@ -477,10 +499,10 @@ module Backup
       Gitlab.config.backup.upload.connection&.provider&.downcase == 'google'
     end
 
-    def repository_backup_strategy(incremental)
+    def repository_backup_strategy
       max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
       max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
-      Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
+      Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
     end
 
     def puts_time(msg)

spec/lib/backup/manager_spec.rb

@@ -145,6 +145,7 @@ RSpec.describe Backup::Manager do
   end
 
   describe '#create' do
+    let(:incremental_env) { 'false' }
     let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz task2.tar.gz} }
     let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' }
     let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
@@ -166,6 +167,7 @@ RSpec.describe Backup::Manager do
     end
 
     before do
+      stub_env('INCREMENTAL', incremental_env)
       allow(ActiveRecord::Base.connection).to receive(:reconnect!)
       allow(Gitlab::BackupLogger).to receive(:info)
       allow(Kernel).to receive(:system).and_return(true)
@@ -561,6 +563,158 @@ RSpec.describe Backup::Manager do
         end
       end
     end
+
+    context 'incremental' do
+      let(:incremental_env) { 'true' }
+      let(:gitlab_version) { Gitlab::VERSION }
+      let(:tar_file) { "1546300800_2019_01_01_#{gitlab_version}_gitlab_backup.tar" }
+      let(:backup_information) do
+        {
+          backup_created_at: Time.zone.parse('2019-01-01'),
+          gitlab_version: gitlab_version
+        }
+      end
+
+      context 'when there are no backup files in the directory' do
+        before do
+          allow(Dir).to receive(:glob).and_return([])
+        end
+
+        it 'fails the operation and prints an error' do
+          expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+          expect(progress).to have_received(:puts)
+            .with(a_string_matching('No backups found'))
+        end
+      end
+
+      context 'when there are two backup files in the directory and BACKUP variable is not set' do
+        before do
+          allow(Dir).to receive(:glob).and_return(
+            [
+              '1451606400_2016_01_01_1.2.3_gitlab_backup.tar',
+              '1451520000_2015_12_31_gitlab_backup.tar'
+            ]
+          )
+        end
+
+        it 'prints the list of available backups' do
+          expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+          expect(progress).to have_received(:puts)
+            .with(a_string_matching('1451606400_2016_01_01_1.2.3\n 1451520000_2015_12_31'))
+        end
+
+        it 'fails the operation and prints an error' do
+          expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+          expect(progress).to have_received(:puts)
+            .with(a_string_matching('Found more than one backup'))
+        end
+      end
+
+      context 'when BACKUP variable is set to a non-existing file' do
+        before do
+          allow(Dir).to receive(:glob).and_return(
+            [
+              '1451606400_2016_01_01_gitlab_backup.tar'
+            ]
+          )
+          allow(File).to receive(:exist?).and_return(false)
+
+          stub_env('BACKUP', 'wrong')
+        end
+
+        it 'fails the operation and prints an error' do
+          expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+          expect(File).to have_received(:exist?).with('wrong_gitlab_backup.tar')
+          expect(progress).to have_received(:puts)
+            .with(a_string_matching('The backup file wrong_gitlab_backup.tar does not exist'))
+        end
+      end
+
+      context 'when BACKUP variable is set to a correct file' do
+        let(:tar_cmdline) { %w{tar -xf 1451606400_2016_01_01_1.2.3_gitlab_backup.tar} }
+
+        before do
+          allow(Gitlab::BackupLogger).to receive(:info)
+          allow(Dir).to receive(:glob).and_return(
+            [
+              '1451606400_2016_01_01_1.2.3_gitlab_backup.tar'
+            ]
+          )
+          allow(File).to receive(:exist?).and_return(true)
+          allow(Kernel).to receive(:system).and_return(true)
+
+          stub_env('BACKUP', '/ignored/path/1451606400_2016_01_01_1.2.3')
+        end
+
+        it 'unpacks the file' do
+          subject.create # rubocop:disable Rails/SaveBang
+
+          expect(Kernel).to have_received(:system).with(*tar_cmdline)
+        end
+
+        context 'tar fails' do
+          before do
+            expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false)
+          end
+
+          it 'logs a failure' do
+            expect do
+              subject.create # rubocop:disable Rails/SaveBang
+            end.to raise_error(SystemExit)
+
+            expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Unpacking backup failed')
+          end
+        end
+
+        context 'on version mismatch' do
+          let(:backup_information) do
+            {
+              backup_created_at: Time.zone.parse('2019-01-01'),
+              gitlab_version: "not #{gitlab_version}"
+            }
+          end
+
+          it 'stops the process' do
+            expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+            expect(progress).to have_received(:puts)
+              .with(a_string_matching('GitLab version mismatch'))
+          end
+        end
+      end
+
+      context 'when there is a non-tarred backup in the directory' do
+        before do
+          allow(Dir).to receive(:glob).and_return(
+            [
+              'backup_information.yml'
+            ]
+          )
+          allow(File).to receive(:exist?).and_return(true)
+        end
+
+        it 'selects the non-tarred backup to restore from' do
+          subject.create # rubocop:disable Rails/SaveBang
+
+          expect(progress).to have_received(:puts)
+            .with(a_string_matching('Non tarred backup found '))
+        end
+
+        context 'on version mismatch' do
+          let(:backup_information) do
+            {
+              backup_created_at: Time.zone.parse('2019-01-01'),
+              gitlab_version: "not #{gitlab_version}"
+            }
+          end
+
+          it 'stops the process' do
+            expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang
+            expect(progress).to have_received(:puts)
+              .with(a_string_matching('GitLab version mismatch'))
+          end
+        end
+      end
+    end
   end
 
   describe '#restore' do