Merge branch '5353-geo-store-checksum-hashes-as-binary-on-the-tracking-database' into 'master'

Geo - Store checksum hashes as binary on the tracking database

Closes #5353

See merge request gitlab-org/gitlab-ee!5039

ee/app/finders/geo/project_registry_finder.rb

@@ -175,8 +175,8 @@ module Geo
       Geo::ProjectRegistry
         .joins(fdw_inner_join_repository_state)
         .where(
-          local_registry_table[:repository_verification_checksum].eq(nil).or(
-            local_registry_table[:wiki_verification_checksum].eq(nil)
+          local_registry_table[:repository_verification_checksum_sha].eq(nil).or(
+            local_registry_table[:wiki_verification_checksum_sha].eq(nil)
           )
         )
         .where(
@@ -308,8 +308,8 @@ module Geo
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of registries that need verification
     def legacy_find_registries_to_verify(batch_size:)
       registries = Geo::ProjectRegistry.where(
-        local_registry_table[:repository_verification_checksum].eq(nil).or(
-          local_registry_table[:wiki_verification_checksum].eq(nil)
+        local_registry_table[:repository_verification_checksum_sha].eq(nil).or(
+          local_registry_table[:wiki_verification_checksum_sha].eq(nil)
         )
       ).pluck(:project_id)
 

ee/app/models/geo/project_registry.rb

 class Geo::ProjectRegistry < Geo::BaseRegistry
   include ::EachBatch
   include ::IgnorableColumn
+  include ::ShaAttribute
 
   ignore_column :last_repository_verification_at
   ignore_column :last_repository_verification_failed
   ignore_column :last_wiki_verification_at
   ignore_column :last_wiki_verification_failed
+  ignore_column :repository_verification_checksum
+  ignore_column :wiki_verification_checksum
+
+  sha_attribute :repository_verification_checksum_sha
+  sha_attribute :wiki_verification_checksum_sha
 
   belongs_to :project
 
@@ -16,8 +22,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   scope :synced_wikis, -> { where(resync_wiki: false) }
   scope :failed_repos, -> { where(arel_table[:repository_retry_count].gt(0)) }
   scope :failed_wikis, -> { where(arel_table[:wiki_retry_count].gt(0)) }
-  scope :verified_repos, -> { where.not(repository_verification_checksum: nil) }
-  scope :verified_wikis, -> { where.not(wiki_verification_checksum: nil) }
+  scope :verified_repos, -> { where.not(repository_verification_checksum_sha: nil) }
+  scope :verified_wikis, -> { where.not(wiki_verification_checksum_sha: nil) }
   scope :verification_failed_repos, -> { where.not(last_repository_verification_failure: nil) }
   scope :verification_failed_wikis, -> { where.not(last_wiki_verification_failure: nil) }
 

ee/app/services/geo/base_sync_service.rb

@@ -122,7 +122,7 @@ module Geo
         attrs["force_to_redownload_#{type}"] = false
 
         # Indicate that repository verification needs to be done again
-        attrs["#{type}_verification_checksum"] = nil
+        attrs["#{type}_verification_checksum_sha"] = nil
         attrs["last_#{type}_verification_failure"] = nil
       end
 

ee/app/services/geo/repository_verify_secondary_service.rb

@@ -36,7 +36,7 @@ module Geo
     end
 
     def secondary_checksum
-      registry.public_send("#{type}_verification_checksum") # rubocop:disable GitlabSecurity/PublicSend
+      registry.public_send("#{type}_verification_checksum_sha") # rubocop:disable GitlabSecurity/PublicSend
     end
 
     def verify_checksum
@@ -61,7 +61,7 @@ module Geo
 
     def update_registry!(checksum: nil, failure: nil, exception: nil, details: {})
       attrs = {
-        "#{type}_verification_checksum"     => checksum,
+        "#{type}_verification_checksum_sha" => checksum,
         "last_#{type}_verification_failure" => failure
       }
 

ee/db/geo/migrate/20180321144947_change_repository_verification_checksum_to_sha.rb

+class ChangeRepositoryVerificationChecksumToSha < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def up
+    add_column :project_registry, :repository_verification_checksum_sha, :binary
+    add_column :project_registry, :wiki_verification_checksum_sha, :binary
+  end
+
+  def down
+    remove_column :project_registry, :repository_verification_checksum_sha
+    remove_column :project_registry, :wiki_verification_checksum_sha
+  end
+end

ee/db/geo/migrate/20180327071612_add_partial_index_to_project_registy_checksum_sha_columns.rb

+class AddPartialIndexToProjectRegistyChecksumShaColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  REPOSITORY_INDEX_NAME = 'idx_project_registry_on_repository_checksum_sha_partial'
+  WIKI_INDEX_NAME = 'idx_project_registry_on_wiki_checksum_sha_partial'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:project_registry, :repository_verification_checksum_sha, name: REPOSITORY_INDEX_NAME, where: 'repository_verification_checksum_sha IS NULL')
+    add_concurrent_index(:project_registry, :wiki_verification_checksum_sha, name: WIKI_INDEX_NAME, where: 'wiki_verification_checksum_sha IS NULL')
+  end
+
+  def down
+    remove_concurrent_index_by_name(:project_registry, REPOSITORY_INDEX_NAME)
+    remove_concurrent_index_by_name(:project_registry, WIKI_INDEX_NAME)
+  end
+end

ee/db/geo/post_migrate/20180326171626_remove_old_repository_verification_checksum_from_geo_project_registry.rb

+class RemoveOldRepositoryVerificationChecksumFromGeoProjectRegistry < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    remove_column :project_registry, :repository_verification_checksum
+    remove_column :project_registry, :wiki_verification_checksum
+  end
+
+  def down
+    add_column :project_registry, :repository_verification_checksum, :string
+    add_column :project_registry, :wiki_verification_checksum, :string
+  end
+end

ee/db/geo/schema.rb

@@ -65,10 +65,10 @@ ActiveRecord::Schema.define(version: 20180331055706) do
     t.boolean "force_to_redownload_wiki"
     t.string "last_repository_sync_failure"
     t.string "last_wiki_sync_failure"
-    t.string "repository_verification_checksum"
     t.string "last_repository_verification_failure"
-    t.string "wiki_verification_checksum"
     t.string "last_wiki_verification_failure"
+    t.binary "repository_verification_checksum_sha"
+    t.binary "wiki_verification_checksum_sha"
   end
 
   add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
@@ -77,10 +77,10 @@ ActiveRecord::Schema.define(version: 20180331055706) do
   add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_failure_partial", where: "(last_wiki_verification_failure IS NOT NULL)", using: :btree
   add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
   add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
-  add_index "project_registry", ["repository_verification_checksum"], name: "idx_project_registry_on_repository_checksum_partial", where: "(repository_verification_checksum IS NULL)", using: :btree
+  add_index "project_registry", ["repository_verification_checksum_sha"], name: "idx_project_registry_on_repository_checksum_sha_partial", where: "(repository_verification_checksum_sha IS NULL)", using: :btree
   add_index "project_registry", ["resync_repository"], name: "index_project_registry_on_resync_repository", using: :btree
   add_index "project_registry", ["resync_wiki"], name: "index_project_registry_on_resync_wiki", using: :btree
   add_index "project_registry", ["wiki_retry_at"], name: "index_project_registry_on_wiki_retry_at", using: :btree
-  add_index "project_registry", ["wiki_verification_checksum"], name: "idx_project_registry_on_wiki_checksum_partial", where: "(wiki_verification_checksum IS NULL)", using: :btree
+  add_index "project_registry", ["wiki_verification_checksum_sha"], name: "idx_project_registry_on_wiki_checksum_sha_partial", where: "(wiki_verification_checksum_sha IS NULL)", using: :btree
 
 end

ee/lib/gitlab/geo/log_cursor/daemon.rb

@@ -104,7 +104,7 @@ module Gitlab
 
         def handle_repository_updated_event(event, created_at)
           registry = find_or_initialize_registry(event.project_id,
-            "resync_#{event.source}" => true, "#{event.source}_verification_checksum" => nil)
+            "resync_#{event.source}" => true, "#{event.source}_verification_checksum_sha" => nil)
 
           registry.save!
 

ee/spec/factories/geo/project_registry.rb

@@ -74,32 +74,32 @@ FactoryBot.define do
     end
 
     trait :repository_verified do
-      repository_verification_checksum 'f079a831cab27bcda7d81cd9b48296d0c3dd92ee'
+      repository_verification_checksum_sha 'f079a831cab27bcda7d81cd9b48296d0c3dd92ee'
       last_repository_verification_failure nil
     end
 
     trait :repository_verification_failed do
-      repository_verification_checksum nil
+      repository_verification_checksum_sha nil
       last_repository_verification_failure 'Repository checksum did not match'
     end
 
     trait :repository_verification_outdated do
-      repository_verification_checksum nil
+      repository_verification_checksum_sha nil
       last_repository_verification_failure nil
     end
 
     trait :wiki_verified do
-      wiki_verification_checksum 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
+      wiki_verification_checksum_sha 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
       last_wiki_verification_failure nil
     end
 
     trait :wiki_verification_failed do
-      wiki_verification_checksum nil
+      wiki_verification_checksum_sha nil
       last_wiki_verification_failure 'Wiki checksum did not match'
     end
 
     trait :wiki_verification_outdated do
-      wiki_verification_checksum nil
+      wiki_verification_checksum_sha nil
       last_wiki_verification_failure nil
     end
   end

ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb

@@ -128,7 +128,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
         it 'resets the repository verification checksum' do
           daemon.run_once!
 
-          expect(registry.reload).to have_attributes(resync_repository: true, repository_verification_checksum: nil)
+          expect(registry.reload).to have_attributes(resync_repository: true, repository_verification_checksum_sha: nil)
         end
       end
 
@@ -148,7 +148,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
         it 'resets the wiki verification checksum' do
           daemon.run_once!
 
-          expect(registry.reload).to have_attributes(resync_wiki: true, wiki_verification_checksum: nil)
+          expect(registry.reload).to have_attributes(resync_wiki: true, wiki_verification_checksum_sha: nil)
         end
       end
 

ee/spec/services/geo/repository_sync_service_spec.rb

@@ -160,10 +160,10 @@ describe Geo::RepositorySyncService do
           expect(registry.last_repository_successful_sync_at).not_to be_nil
         end
 
-        it 'resets the repository_verification_checksum' do
+        it 'resets the repository_verification_checksum_sha' do
           subject.execute
 
-          expect(registry.repository_verification_checksum).to be_nil
+          expect(registry.repository_verification_checksum_sha).to be_nil
         end
 
         it 'resets the last_repository_verification_failure' do

ee/spec/services/geo/repository_verify_secondary_service_spec.rb

@@ -42,7 +42,7 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
 
     it 'does not verify the checksum if the checksums already match' do
       repository_state.assign_attributes("#{type}_verification_checksum" => 'my_checksum')
-      registry.assign_attributes("#{type}_verification_checksum" => 'my_checksum')
+      registry.assign_attributes("#{type}_verification_checksum_sha" => 'my_checksum')
 
       expect(Gitlab::Git::Checksum).not_to receive(:new).with(storage, relative_path)
 
@@ -53,7 +53,7 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
       expect(Gitlab::Git::Checksum).to receive(:new).with(storage, relative_path) { checksum }
       expect(checksum).to receive(:calculate).and_return('my_checksum')
 
-      expect { service.execute }.to change(registry, "#{type}_verification_checksum")
+      expect { service.execute }.to change(registry, "#{type}_verification_checksum_sha")
         .from(nil).to('my_checksum')
     end
 

ee/spec/services/geo/wiki_sync_service_spec.rb

@@ -139,10 +139,10 @@ RSpec.describe Geo::WikiSyncService do
           expect(registry.last_wiki_successful_sync_at).not_to be_nil
         end
 
-        it 'resets the wiki_verification_checksum' do
+        it 'resets the wiki_verification_checksum_sha' do
           subject.execute
 
-          expect(registry.wiki_verification_checksum).to be_nil
+          expect(registry.wiki_verification_checksum_sha).to be_nil
         end
 
         it 'resets the last_wiki_verification_failure' do