Commit 8fcb2930 authored by Brett Walker's avatar Brett Walker Committed by Stan Hu

Geo - Store checksum hashes as binary on the tracking database

parent 5cd33f42
......@@ -175,8 +175,8 @@ module Geo
Geo::ProjectRegistry
.joins(fdw_inner_join_repository_state)
.where(
local_registry_table[:repository_verification_checksum].eq(nil).or(
local_registry_table[:wiki_verification_checksum].eq(nil)
local_registry_table[:repository_verification_checksum_sha].eq(nil).or(
local_registry_table[:wiki_verification_checksum_sha].eq(nil)
)
)
.where(
......@@ -308,8 +308,8 @@ module Geo
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of registries that need verification
def legacy_find_registries_to_verify(batch_size:)
registries = Geo::ProjectRegistry.where(
local_registry_table[:repository_verification_checksum].eq(nil).or(
local_registry_table[:wiki_verification_checksum].eq(nil)
local_registry_table[:repository_verification_checksum_sha].eq(nil).or(
local_registry_table[:wiki_verification_checksum_sha].eq(nil)
)
).pluck(:project_id)
......
class Geo::ProjectRegistry < Geo::BaseRegistry
include ::EachBatch
include ::IgnorableColumn
include ::ShaAttribute
ignore_column :last_repository_verification_at
ignore_column :last_repository_verification_failed
ignore_column :last_wiki_verification_at
ignore_column :last_wiki_verification_failed
ignore_column :repository_verification_checksum
ignore_column :wiki_verification_checksum
sha_attribute :repository_verification_checksum_sha
sha_attribute :wiki_verification_checksum_sha
belongs_to :project
......@@ -16,8 +22,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
scope :synced_wikis, -> { where(resync_wiki: false) }
scope :failed_repos, -> { where(arel_table[:repository_retry_count].gt(0)) }
scope :failed_wikis, -> { where(arel_table[:wiki_retry_count].gt(0)) }
scope :verified_repos, -> { where.not(repository_verification_checksum: nil) }
scope :verified_wikis, -> { where.not(wiki_verification_checksum: nil) }
scope :verified_repos, -> { where.not(repository_verification_checksum_sha: nil) }
scope :verified_wikis, -> { where.not(wiki_verification_checksum_sha: nil) }
scope :verification_failed_repos, -> { where.not(last_repository_verification_failure: nil) }
scope :verification_failed_wikis, -> { where.not(last_wiki_verification_failure: nil) }
......
......@@ -122,7 +122,7 @@ module Geo
attrs["force_to_redownload_#{type}"] = false
# Indicate that repository verification needs to be done again
attrs["#{type}_verification_checksum"] = nil
attrs["#{type}_verification_checksum_sha"] = nil
attrs["last_#{type}_verification_failure"] = nil
end
......
......@@ -36,7 +36,7 @@ module Geo
end
def secondary_checksum
registry.public_send("#{type}_verification_checksum") # rubocop:disable GitlabSecurity/PublicSend
registry.public_send("#{type}_verification_checksum_sha") # rubocop:disable GitlabSecurity/PublicSend
end
def verify_checksum
......@@ -61,7 +61,7 @@ module Geo
def update_registry!(checksum: nil, failure: nil, exception: nil, details: {})
attrs = {
"#{type}_verification_checksum" => checksum,
"#{type}_verification_checksum_sha" => checksum,
"last_#{type}_verification_failure" => failure
}
......
class ChangeRepositoryVerificationChecksumToSha < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def up
add_column :project_registry, :repository_verification_checksum_sha, :binary
add_column :project_registry, :wiki_verification_checksum_sha, :binary
end
def down
remove_column :project_registry, :repository_verification_checksum_sha
remove_column :project_registry, :wiki_verification_checksum_sha
end
end
class AddPartialIndexToProjectRegistyChecksumShaColumns < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
REPOSITORY_INDEX_NAME = 'idx_project_registry_on_repository_checksum_sha_partial'
WIKI_INDEX_NAME = 'idx_project_registry_on_wiki_checksum_sha_partial'
disable_ddl_transaction!
def up
add_concurrent_index(:project_registry, :repository_verification_checksum_sha, name: REPOSITORY_INDEX_NAME, where: 'repository_verification_checksum_sha IS NULL')
add_concurrent_index(:project_registry, :wiki_verification_checksum_sha, name: WIKI_INDEX_NAME, where: 'wiki_verification_checksum_sha IS NULL')
end
def down
remove_concurrent_index_by_name(:project_registry, REPOSITORY_INDEX_NAME)
remove_concurrent_index_by_name(:project_registry, WIKI_INDEX_NAME)
end
end
class RemoveOldRepositoryVerificationChecksumFromGeoProjectRegistry < ActiveRecord::Migration
DOWNTIME = false
def up
remove_column :project_registry, :repository_verification_checksum
remove_column :project_registry, :wiki_verification_checksum
end
def down
add_column :project_registry, :repository_verification_checksum, :string
add_column :project_registry, :wiki_verification_checksum, :string
end
end
......@@ -65,10 +65,10 @@ ActiveRecord::Schema.define(version: 20180331055706) do
t.boolean "force_to_redownload_wiki"
t.string "last_repository_sync_failure"
t.string "last_wiki_sync_failure"
t.string "repository_verification_checksum"
t.string "last_repository_verification_failure"
t.string "wiki_verification_checksum"
t.string "last_wiki_verification_failure"
t.binary "repository_verification_checksum_sha"
t.binary "wiki_verification_checksum_sha"
end
add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
......@@ -77,10 +77,10 @@ ActiveRecord::Schema.define(version: 20180331055706) do
add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_failure_partial", where: "(last_wiki_verification_failure IS NOT NULL)", using: :btree
add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
add_index "project_registry", ["repository_verification_checksum"], name: "idx_project_registry_on_repository_checksum_partial", where: "(repository_verification_checksum IS NULL)", using: :btree
add_index "project_registry", ["repository_verification_checksum_sha"], name: "idx_project_registry_on_repository_checksum_sha_partial", where: "(repository_verification_checksum_sha IS NULL)", using: :btree
add_index "project_registry", ["resync_repository"], name: "index_project_registry_on_resync_repository", using: :btree
add_index "project_registry", ["resync_wiki"], name: "index_project_registry_on_resync_wiki", using: :btree
add_index "project_registry", ["wiki_retry_at"], name: "index_project_registry_on_wiki_retry_at", using: :btree
add_index "project_registry", ["wiki_verification_checksum"], name: "idx_project_registry_on_wiki_checksum_partial", where: "(wiki_verification_checksum IS NULL)", using: :btree
add_index "project_registry", ["wiki_verification_checksum_sha"], name: "idx_project_registry_on_wiki_checksum_sha_partial", where: "(wiki_verification_checksum_sha IS NULL)", using: :btree
end
......@@ -104,7 +104,7 @@ module Gitlab
def handle_repository_updated_event(event, created_at)
registry = find_or_initialize_registry(event.project_id,
"resync_#{event.source}" => true, "#{event.source}_verification_checksum" => nil)
"resync_#{event.source}" => true, "#{event.source}_verification_checksum_sha" => nil)
registry.save!
......
......@@ -74,32 +74,32 @@ FactoryBot.define do
end
trait :repository_verified do
repository_verification_checksum 'f079a831cab27bcda7d81cd9b48296d0c3dd92ee'
repository_verification_checksum_sha 'f079a831cab27bcda7d81cd9b48296d0c3dd92ee'
last_repository_verification_failure nil
end
trait :repository_verification_failed do
repository_verification_checksum nil
repository_verification_checksum_sha nil
last_repository_verification_failure 'Repository checksum did not match'
end
trait :repository_verification_outdated do
repository_verification_checksum nil
repository_verification_checksum_sha nil
last_repository_verification_failure nil
end
trait :wiki_verified do
wiki_verification_checksum 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
wiki_verification_checksum_sha 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
last_wiki_verification_failure nil
end
trait :wiki_verification_failed do
wiki_verification_checksum nil
wiki_verification_checksum_sha nil
last_wiki_verification_failure 'Wiki checksum did not match'
end
trait :wiki_verification_outdated do
wiki_verification_checksum nil
wiki_verification_checksum_sha nil
last_wiki_verification_failure nil
end
end
......
......@@ -128,7 +128,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
it 'resets the repository verification checksum' do
daemon.run_once!
expect(registry.reload).to have_attributes(resync_repository: true, repository_verification_checksum: nil)
expect(registry.reload).to have_attributes(resync_repository: true, repository_verification_checksum_sha: nil)
end
end
......@@ -148,7 +148,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
it 'resets the wiki verification checksum' do
daemon.run_once!
expect(registry.reload).to have_attributes(resync_wiki: true, wiki_verification_checksum: nil)
expect(registry.reload).to have_attributes(resync_wiki: true, wiki_verification_checksum_sha: nil)
end
end
......
......@@ -160,10 +160,10 @@ describe Geo::RepositorySyncService do
expect(registry.last_repository_successful_sync_at).not_to be_nil
end
it 'resets the repository_verification_checksum' do
it 'resets the repository_verification_checksum_sha' do
subject.execute
expect(registry.repository_verification_checksum).to be_nil
expect(registry.repository_verification_checksum_sha).to be_nil
end
it 'resets the last_repository_verification_failure' do
......
......@@ -42,7 +42,7 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
it 'does not verify the checksum if the checksums already match' do
repository_state.assign_attributes("#{type}_verification_checksum" => 'my_checksum')
registry.assign_attributes("#{type}_verification_checksum" => 'my_checksum')
registry.assign_attributes("#{type}_verification_checksum_sha" => 'my_checksum')
expect(Gitlab::Git::Checksum).not_to receive(:new).with(storage, relative_path)
......@@ -53,7 +53,7 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
expect(Gitlab::Git::Checksum).to receive(:new).with(storage, relative_path) { checksum }
expect(checksum).to receive(:calculate).and_return('my_checksum')
expect { service.execute }.to change(registry, "#{type}_verification_checksum")
expect { service.execute }.to change(registry, "#{type}_verification_checksum_sha")
.from(nil).to('my_checksum')
end
......
......@@ -139,10 +139,10 @@ RSpec.describe Geo::WikiSyncService do
expect(registry.last_wiki_successful_sync_at).not_to be_nil
end
it 'resets the wiki_verification_checksum' do
it 'resets the wiki_verification_checksum_sha' do
subject.execute
expect(registry.wiki_verification_checksum).to be_nil
expect(registry.wiki_verification_checksum_sha).to be_nil
end
it 'resets the last_wiki_verification_failure' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment