Merge branch 'incremental_repository_backup' into 'master'

Create feature flag for incremental repository backups

See merge request gitlab-org/gitlab!79589

config/feature_flags/development/incremental_repository_backup.yml

+---
+name: incremental_repository_backup
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79589
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/355945
+milestone: '14.9'
+type: development
+group: group::gitaly
+default_enabled: false

doc/raketasks/backup_restore.md

@@ -1854,3 +1854,22 @@ To enable it:
 ```ruby
 Feature.enable(:gitaly_backup)
 ```
+
+### Incremental repository backups
+
+> Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`.
+On GitLab.com, this feature is not available.
+This feature is not ready for production use.
+
+Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup
+bundle for each repository. Because incremental backups require access to the previous backup, you can't use incremental
+backups with tar files.
+
+To create an incremental backup, run:
+
+```shell
+sudo gitlab-backup create SKIP=tar INCREMENTAL=yes
+```

lib/backup/gitaly_backup.rb

@@ -9,10 +9,13 @@ module Backup
     # @param [StringIO] progress IO interface to output progress
     # @param [Integer] max_parallelism max parallelism when running backups
     # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism)
-    def initialize(progress, max_parallelism: nil, storage_parallelism: nil)
+    # @param [String] backup_id unique identifier for the backup
+    def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil)
       @progress = progress
       @max_parallelism = max_parallelism
       @storage_parallelism = storage_parallelism
+      @incremental = incremental
+      @backup_id = backup_id
     end
 
     def start(type, backup_repos_path)
@@ -30,6 +33,13 @@ module Backup
       args = []
       args += ['-parallel', @max_parallelism.to_s] if @max_parallelism
       args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism
+      if Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml)
+        args += ['-layout', 'pointer']
+        if type == :create
+          args += ['-incremental'] if @incremental
+          args += ['-id', @backup_id] if @backup_id
+        end
+      end
 
       @input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args)
 

lib/backup/manager.rb

@@ -21,6 +21,7 @@ module Backup
       max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
       max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
       force = ENV['force'] == 'yes'
+      incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false)
 
       @definitions = definitions || {
         'db' => TaskDefinition.new(
@@ -32,7 +33,7 @@ module Backup
           destination_path: 'repositories',
           destination_optional: true,
           task: Repositories.new(progress,
-                                 strategy: repository_backup_strategy,
+                                 strategy: repository_backup_strategy(incremental),
                                  max_concurrency: max_concurrency,
                                  max_storage_concurrency: max_storage_concurrency)
         ),
@@ -481,11 +482,11 @@ module Backup
       Gitlab.config.backup.upload.connection&.provider&.downcase == 'google'
     end
 
-    def repository_backup_strategy
+    def repository_backup_strategy(incremental)
       if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
         max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
-        Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
+        Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
       else
         Backup::GitalyRpcBackup.new(progress)
       end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -176,8 +176,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
           expect(exit_status).to eq(0)
           expect(tar_contents).to match(user_backup_path)
-          expect(tar_contents).to match("#{user_backup_path}/custom_hooks.tar")
-          expect(tar_contents).to match("#{user_backup_path}.bundle")
+          expect(tar_contents).to match("#{user_backup_path}/.+/001.custom_hooks.tar")
+          expect(tar_contents).to match("#{user_backup_path}/.+/001.bundle")
         end
 
         it 'restores files correctly' do
@@ -360,14 +360,14 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(exit_status).to eq(0)
 
           [
-            "#{project_a.disk_path}.bundle",
-            "#{project_a.disk_path}.wiki.bundle",
-            "#{project_a.disk_path}.design.bundle",
-            "#{project_b.disk_path}.bundle",
-            "#{project_snippet_a.disk_path}.bundle",
-            "#{project_snippet_b.disk_path}.bundle"
+            "#{project_a.disk_path}/.+/001.bundle",
+            "#{project_a.disk_path}.wiki/.+/001.bundle",
+            "#{project_a.disk_path}.design/.+/001.bundle",
+            "#{project_b.disk_path}/.+/001.bundle",
+            "#{project_snippet_a.disk_path}/.+/001.bundle",
+            "#{project_snippet_b.disk_path}/.+/001.bundle"
           ].each do |repo_name|
-            expect(tar_lines.grep(/#{repo_name}/).size).to eq 1
+            expect(tar_lines).to include(a_string_matching(repo_name))
           end
         end
 
@@ -428,7 +428,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         expect(::Backup::Repositories).to receive(:new)
           .with(anything, strategy: anything, max_concurrency: 5, max_storage_concurrency: 2)
           .and_call_original
-        expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2).and_call_original
+        expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2, incremental: false).and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end