Add Backup and Restore tasks for Terraform States

Reuses standard file upload storage to cover Terraform
states backup

Adds a seed for generating terraform state files
under each project

Manual testing (in local GDK env):

- Verfied seed addition via `bin/rake db:seed_fu FILTER=terraform`
- Verified seed addition is repeatable, with no-ops if versions
  pre-exist
- Created backup: `bin/rake gitlab:backup:create`
- Verified output shows up:

```
2021-06-05 23:48:54 +0530 -- Dumping terraform states ...
2021-06-05 23:48:54 +0530 -- done
```

- Verified tar contains terraform_state.tar.gz
- Verified terraform_state.tar.gz carries content
- Destroyed all records: `Terraform::State.destroy_all`
- Deleted all files: `rm -rf shared/terraform_state/*`
- Performed restore: `bin/rake gitlab:backup:restore`
- Verified `Terraform::State.all` shows back older records
- Verified association of builds to some of these records
- Verified `shared/terraform_state/` is repopulated
- Verified `Terraform::StateVersion.each do |sv| pp sv.file.read end`
  works and shows the seed-added data
- Verified skipping of `terraform_state` keyword works:

```
2021-06-06 00:11:33 +0530 -- Dumping terraform states ...
2021-06-06 00:11:33 +0530 -- [SKIPPED]
```

Verified with same workflow as above (backup, destroy, delete,
then restore) that skipped backup variant does not contain
`terraform_state.tar.gz` and does not show up those files after
restore

Miscellany:

- Adds a missing test for LFS backups
- Adds File operation fixes to neighboring tests
  so when they fail RSpec is able to show diffs
- Modified test validates unknown SKIP values not
  breaking backups
  - This is useful to catch any regressions during
    upgrades where pre-installs scripts may refer
    to a new SKIP value that isn't recognized by
    the old version

Changelog: added

.gitlab/ci/rails.gitlab-ci.yml

@@ -463,7 +463,7 @@ db:backup_and_restore:
   script:
     - . scripts/prepare_build.sh
     - bundle exec rake db:drop db:create db:structure:load db:seed_fu
-    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
+    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry}
     - bundle exec rake gitlab:backup:create
     - date
     - bundle exec rake gitlab:backup:restore

db/fixtures/development/31_terraform_state.rb

+# frozen_string_literal: true
+
+TERRAFORM_FILE_VERSION = 1
+
+# Create sample terraform states in existing projects
+Gitlab::Seeder.quiet do
+  tfdata = {terraform_version: '0.14.1'}.to_json
+
+  Project.not_mass_generated.find_each do |project|
+    # Create as the project's creator
+    user = project.creator
+    # Set a build job source, if one exists for the project
+    build = project.builds.last
+
+    remote_state_handler = ::Terraform::RemoteStateHandler.new(project, user, name: project.path, lock_id: nil)
+
+    remote_state_handler.handle_with_lock do |state|
+      # Upload a file if a version does not already exist
+      state.update_file!(CarrierWaveStringFile.new(tfdata), version: TERRAFORM_FILE_VERSION, build: build) if state.latest_version.nil?
+    end
+
+    # rubocop:disable Rails/Output
+    print '.'
+    # rubocop:enable Rails/Output
+  end
+end

doc/raketasks/backup_restore.md

@@ -58,6 +58,7 @@ including:
 - CI/CD job output logs
 - CI/CD job artifacts
 - LFS objects
+- Terraform states
 - Container Registry images
 - GitLab Pages content
 - Snippets
@@ -65,7 +66,6 @@ including:
 
 Backups do not include:
 
-- [Terraform state files](../administration/terraform_state.md)
 - [Package registry files](../administration/packages/index.md)
 - [Mattermost data](https://docs.mattermost.com/administration/config-settings.html#file-storage)
 
@@ -276,6 +276,7 @@ You can exclude specific directories from the backup by adding the environment v
 - `builds` (CI job output logs)
 - `artifacts` (CI job artifacts)
 - `lfs` (LFS objects)
+- `terraform_state` (Terraform states)
 - `registry` (Container Registry images)
 - `pages` (Pages content)
 - `repositories` (Git repositories data)

lib/backup/manager.rb

@@ -2,7 +2,7 @@
 
 module Backup
   class Manager
-    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
+    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry].freeze
     FOLDERS_TO_BACKUP = %w[repositories db].freeze
     FILE_NAME_SUFFIX = '_gitlab_backup.tar'
 

lib/backup/terraform_state.rb

+# frozen_string_literal: true
+
+module Backup
+  class TerraformState < Backup::Files
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
+      super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
+    end
+  end
+end

lib/tasks/gitlab/backup.rake

@@ -16,6 +16,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:artifacts:create'].invoke
       Rake::Task['gitlab:backup:pages:create'].invoke
       Rake::Task['gitlab:backup:lfs:create'].invoke
+      Rake::Task['gitlab:backup:terraform_state:create'].invoke
       Rake::Task['gitlab:backup:registry:create'].invoke
 
       backup = Backup::Manager.new(progress)
@@ -83,6 +84,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
       Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages')
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+      Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state')
       Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
       Rake::Task['gitlab:shell:setup'].invoke
       Rake::Task['cache:clear'].invoke
@@ -254,6 +256,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :terraform_state do
+      task create: :gitlab_environment do
+        puts_time "Dumping terraform states ... ".color(:blue)
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state")
+          puts_time "[SKIPPED]".color(:cyan)
+        else
+          Backup::TerraformState.new(progress).dump
+          puts_time "done".color(:green)
+        end
+      end
+
+      task restore: :gitlab_environment do
+        puts_time "Restoring terraform states ... ".color(:blue)
+        Backup::TerraformState.new(progress).restore
+        puts_time "done".color(:green)
+      end
+    end
+
     namespace :registry do
       task create: :gitlab_environment do
         puts_time "Dumping container registry images ... ".color(:blue)

spec/lib/backup/artifacts_spec.rb

@@ -12,7 +12,7 @@ RSpec.describe Backup::Artifacts do
       Dir.mktmpdir do |tmpdir|
         allow(JobArtifactUploader).to receive(:root) { "#{tmpdir}" }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}")
       end
     end
   end

spec/lib/backup/lfs_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Lfs do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with('/var/lfs-objects').and_return('/var/lfs-objects')
+      allow(File).to receive(:realpath).with('/var/lfs-objects/..').and_return('/var')
+      allow(Settings.lfs).to receive(:storage_path).and_return('/var/lfs-objects')
+    end
+
+    it 'uses the correct lfs dir in tar command', :aggregate_failures do
+      expect(backup.app_files_dir).to eq('/var/lfs-objects')
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end

spec/lib/backup/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Backup::Manager do
   end
 
   describe '#pack' do
-    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml) }
+    let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml) }
     let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' }
     let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
     let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] }
@@ -57,7 +57,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when skipped is set in backup_information.yml' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
       let(:backup_information) do
         {
           backup_created_at: Time.zone.parse('2019-01-01'),
@@ -74,7 +74,7 @@ RSpec.describe Backup::Manager do
     end
 
     context 'when a directory does not exist' do
-      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+      let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
 
       before do
         expect(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'repositories')).and_return(false)

spec/lib/backup/terraform_state_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::TerraformState do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).and_call_original
+      allow(File).to receive(:realpath).with('/var/terraform_state').and_return('/var/terraform_state')
+      allow(File).to receive(:realpath).with('/var/terraform_state/..').and_return('/var')
+      allow(Settings.terraform_state).to receive(:storage_path).and_return('/var/terraform_state')
+    end
+
+    it 'uses the correct storage dir in tar command and excludes tmp', :aggregate_failures do
+      expect(backup.app_files_dir).to eq('/var/terraform_state')
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/terraform_state -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+      expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+      backup.dump
+    end
+  end
+end

spec/lib/backup/uploads_spec.rb

@@ -14,13 +14,14 @@ RSpec.describe Backup::Uploads do
 
         allow(Gitlab.config.uploads).to receive(:storage_path) { tmpdir }
 
-        expect(backup.app_files_dir).to eq("#{tmpdir}/uploads")
+        expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}/uploads")
       end
     end
   end
 
   describe '#dump' do
     before do
+      allow(File).to receive(:realpath).and_call_original
       allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads')
       allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var')
       allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' }

spec/support/helpers/test_env.rb

@@ -148,6 +148,8 @@ module TestEnv
     FileUtils.mkdir_p(backup_path)
     FileUtils.mkdir_p(pages_path)
     FileUtils.mkdir_p(artifacts_path)
+    FileUtils.mkdir_p(lfs_path)
+    FileUtils.mkdir_p(terraform_state_path)
   end
 
   def setup_gitlab_shell
@@ -414,6 +416,14 @@ module TestEnv
     Gitlab.config.artifacts.storage_path
   end
 
+  def lfs_path
+    Gitlab.config.lfs.storage_path
+  end
+
+  def terraform_state_path
+    Gitlab.config.terraform_state.storage_path
+  end
+
   # When no cached assets exist, manually hit the root path to create them
   #
   # Otherwise they'd be created by the first test, often timing out and