Commit 92b7d1bb authored by Valery Sizov's avatar Valery Sizov

GEO: Address review comments

parent 5754557d
class Geo::FileRegistry < Geo::BaseRegistry class Geo::FileRegistry < Geo::BaseRegistry
scope :failed, -> { where(success: false) } scope :failed, -> { where(success: false) }
scope :synced, -> { where(success: true) } scope :synced, -> { where(success: true) }
scope :to_be_retried, -> { where('retry_at is NULL OR retry_at < ?', Time.now) } scope :retry_due, -> { where('retry_at is NULL OR retry_at < ?', Time.now) }
scope :lfs_objects, -> { where(file_type: :lfs) } scope :lfs_objects, -> { where(file_type: :lfs) }
scope :attachments, -> { where(file_type: Geo::FileService::DEFAULT_OBJECT_TYPES) } scope :attachments, -> { where(file_type: Geo::FileService::DEFAULT_OBJECT_TYPES) }
end end
...@@ -15,7 +15,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry ...@@ -15,7 +15,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
where(repository_sync_failed.or(wiki_sync_failed)) where(repository_sync_failed.or(wiki_sync_failed))
end end
def self.to_be_retried def self.retry_due
where( where(
arel_table[:repository_retry_at].lt(Time.now) arel_table[:repository_retry_at].lt(Time.now)
.or(arel_table[:wiki_retry_at].lt(Time.now)) .or(arel_table[:wiki_retry_at].lt(Time.now))
......
...@@ -172,7 +172,12 @@ module Geo ...@@ -172,7 +172,12 @@ module Geo
end end
def disk_path_temp def disk_path_temp
"#{repository.disk_path}_temp" unless @disk_path_temp
random_string = (0...6).map { ('a'..'z').to_a.sample }.join
@disk_path_temp = "#{repository.disk_path}_#{random_string}"
end
@disk_path_temp
end end
def build_temporary_repository def build_temporary_repository
......
...@@ -17,7 +17,6 @@ module Geo ...@@ -17,7 +17,6 @@ module Geo
if redownload if redownload
log_info('Redownloading repository') log_info('Redownloading repository')
clean_up_temporary_repository
fetch_geo_mirror(build_temporary_repository) fetch_geo_mirror(build_temporary_repository)
set_temp_repository_as_main set_temp_repository_as_main
else else
...@@ -37,6 +36,8 @@ module Geo ...@@ -37,6 +36,8 @@ module Geo
log_error('Invalid repository', e) log_error('Invalid repository', e)
registry.update(force_to_redownload_repository: true) registry.update(force_to_redownload_repository: true)
expire_repository_caches expire_repository_caches
ensure
clean_up_temporary_repository if redownload
end end
def expire_repository_caches def expire_repository_caches
......
...@@ -16,7 +16,6 @@ module Geo ...@@ -16,7 +16,6 @@ module Geo
if redownload if redownload
log_info('Redownloading wiki') log_info('Redownloading wiki')
clean_up_temporary_repository
fetch_geo_mirror(build_temporary_repository) fetch_geo_mirror(build_temporary_repository)
set_temp_repository_as_main set_temp_repository_as_main
else else
...@@ -37,6 +36,8 @@ module Geo ...@@ -37,6 +36,8 @@ module Geo
rescue Gitlab::Git::Repository::NoRepository => e rescue Gitlab::Git::Repository::NoRepository => e
log_error('Invalid wiki', e) log_error('Invalid wiki', e)
registry.update(force_to_redownload_wiki: true) registry.update(force_to_redownload_wiki: true)
ensure
clean_up_temporary_repository if redownload
end end
def ssh_url_to_wiki def ssh_url_to_wiki
......
...@@ -33,7 +33,7 @@ module Geo ...@@ -33,7 +33,7 @@ module Geo
def find_failed_objects(batch_size:) def find_failed_objects(batch_size:)
Geo::FileRegistry Geo::FileRegistry
.failed .failed
.to_be_retried .retry_due
.limit(batch_size) .limit(batch_size)
.pluck(:file_id, :file_type) .pluck(:file_id, :file_type)
end end
......
...@@ -33,7 +33,7 @@ module Geo ...@@ -33,7 +33,7 @@ module Geo
def find_project_ids_updated_recently(batch_size:) def find_project_ids_updated_recently(batch_size:)
current_node.project_registries current_node.project_registries
.dirty .dirty
.to_be_retried .retry_due
.order(Gitlab::Database.nulls_first_order(:last_repository_synced_at, :desc)) .order(Gitlab::Database.nulls_first_order(:last_repository_synced_at, :desc))
.limit(batch_size) .limit(batch_size)
.pluck(:project_id) .pluck(:project_id)
......
class AddRetryCountFieldsToRegistries < ActiveRecord::Migration class AddRetryCountFieldsToRegistries < ActiveRecord::Migration
def change include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column :file_registry, :retry_count, :integer add_column :file_registry, :retry_count, :integer
add_column :file_registry, :retry_at, :datetime add_column :file_registry, :retry_at, :datetime
...@@ -10,5 +16,23 @@ class AddRetryCountFieldsToRegistries < ActiveRecord::Migration ...@@ -10,5 +16,23 @@ class AddRetryCountFieldsToRegistries < ActiveRecord::Migration
add_column :project_registry, :wiki_retry_count, :integer add_column :project_registry, :wiki_retry_count, :integer
add_column :project_registry, :wiki_retry_at, :datetime add_column :project_registry, :wiki_retry_at, :datetime
add_column :project_registry, :force_to_redownload_wiki, :boolean add_column :project_registry, :force_to_redownload_wiki, :boolean
# Indecies
add_concurrent_index :file_registry, :retry_at
add_concurrent_index :project_registry, :repository_retry_at
add_concurrent_index :project_registry, :wiki_retry_at
end
def down
remove_column :file_registry, :retry_count, :integer
remove_column :file_registry, :retry_at, :datetime
remove_column :project_registry, :repository_retry_count, :integer
remove_column :project_registry, :repository_retry_at, :datetime
remove_column :project_registry, :force_to_redownload_repository, :boolean
remove_column :project_registry, :wiki_retry_count, :integer
remove_column :project_registry, :wiki_retry_at, :datetime
remove_column :project_registry, :force_to_redownload_wiki, :boolean
end end
end end
...@@ -32,6 +32,7 @@ ActiveRecord::Schema.define(version: 20171101105200) do ...@@ -32,6 +32,7 @@ ActiveRecord::Schema.define(version: 20171101105200) do
add_index "file_registry", ["file_type", "file_id"], name: "index_file_registry_on_file_type_and_file_id", unique: true, using: :btree add_index "file_registry", ["file_type", "file_id"], name: "index_file_registry_on_file_type_and_file_id", unique: true, using: :btree
add_index "file_registry", ["file_type"], name: "index_file_registry_on_file_type", using: :btree add_index "file_registry", ["file_type"], name: "index_file_registry_on_file_type", using: :btree
add_index "file_registry", ["retry_at"], name: "index_file_registry_on_retry_at", using: :btree
add_index "file_registry", ["success"], name: "index_file_registry_on_success", using: :btree add_index "file_registry", ["success"], name: "index_file_registry_on_success", using: :btree
create_table "project_registry", force: :cascade do |t| create_table "project_registry", force: :cascade do |t|
...@@ -54,7 +55,9 @@ ActiveRecord::Schema.define(version: 20171101105200) do ...@@ -54,7 +55,9 @@ ActiveRecord::Schema.define(version: 20171101105200) do
add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
add_index "project_registry", ["last_repository_synced_at"], name: "index_project_registry_on_last_repository_synced_at", using: :btree add_index "project_registry", ["last_repository_synced_at"], name: "index_project_registry_on_last_repository_synced_at", using: :btree
add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
add_index "project_registry", ["resync_repository"], name: "index_project_registry_on_resync_repository", using: :btree add_index "project_registry", ["resync_repository"], name: "index_project_registry_on_resync_repository", using: :btree
add_index "project_registry", ["resync_wiki"], name: "index_project_registry_on_resync_wiki", using: :btree add_index "project_registry", ["resync_wiki"], name: "index_project_registry_on_resync_wiki", using: :btree
add_index "project_registry", ["wiki_retry_at"], name: "index_project_registry_on_wiki_retry_at", using: :btree
end end
...@@ -9,10 +9,10 @@ describe Geo::BaseSyncService do ...@@ -9,10 +9,10 @@ describe Geo::BaseSyncService do
describe '#lease_key' do describe '#lease_key' do
it 'returns a key in the correct pattern' do it 'returns a key in the correct pattern' do
described_class.type = :test allow(described_class).to receive(:type) { :wiki }
allow(project).to receive(:id) { 999 } allow(project).to receive(:id) { 999 }
expect(subject.lease_key).to eq('geo_sync_service:test:999') expect(subject.lease_key).to eq('geo_sync_service:wiki:999')
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment