Commit 49988c12 authored by Valery Sizov's avatar Valery Sizov Committed by Nick Thomas

Resolve "Geo: sync disabled wikis"

parent 7f8db8de
......@@ -1176,6 +1176,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do
t.binary "storage_configuration_digest"
t.integer "repositories_retrying_verification_count"
t.integer "wikis_retrying_verification_count"
t.integer "projects_count"
end
add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree
......
......@@ -178,6 +178,7 @@ Example response:
"job_artifacts_synced_missing_on_primary_count": 0,
"job_artifacts_synced_in_percentage": "0.00%",
"repositories_count": 41,
"projects_count": 41,
"repositories_failed_count": nil,
"repositories_synced_count": nil,
"repositories_synced_in_percentage": "0.00%",
......@@ -242,6 +243,7 @@ Example response:
"job_artifacts_synced_missing_on_primary_count": 0,
"job_artifacts_synced_in_percentage": "50.00%",
"repositories_count": 41,
"projects_count": 41,
"repositories_failed_count": 1,
"repositories_synced_count": 40,
"repositories_synced_in_percentage": "97.56%",
......@@ -283,6 +285,8 @@ Example response:
]
```
Note: fields `wikis_count` and `repositories_count` are deprecated and will be deleted soon. Please use `projects_count` instead.
## Retrieve status about a specific Geo node
```
......@@ -319,6 +323,7 @@ Example response:
"job_artifacts_synced_missing_on_primary_count": 0,
"job_artifacts_synced_in_percentage": "50.00%",
"repositories_count": 41,
"projects_count": 41,
"repositories_failed_count": 1,
"repositories_synced_count": 40,
"repositories_synced_in_percentage": "97.56%",
......@@ -340,7 +345,9 @@ Example response:
}
```
Please note that the `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message.
Note: The `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message.
Note: Fields `wikis_count` and `repositories_count` are deprecated and will be deleted soon. Please use `projects_count` instead.
## Retrieve project sync or verification failures that occurred on the current node
......
module Geo
class ProjectRegistryFinder < RegistryFinder
def count_repositories
def count_projects
current_node.projects.count
end
def count_wikis
current_node.projects.with_wiki_enabled.count
end
def count_synced_repositories
relation =
if selective_sync?
......@@ -194,7 +190,7 @@ module Geo
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
def fdw_find_synced_wikis
Geo::ProjectRegistry.synced_wikis.where(fdw_enabled_wikis)
Geo::ProjectRegistry.synced_wikis
end
# @return [ActiveRecord::Relation<Geo::Fdw::Project>]
......@@ -222,7 +218,7 @@ module Geo
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
def fdw_find_verified_wikis
Geo::ProjectRegistry.verified_wikis.where(fdw_enabled_wikis)
Geo::ProjectRegistry.verified_wikis
end
def fdw_inner_join_repository_state
......@@ -232,20 +228,6 @@ module Geo
.join_sources
end
def fdw_enabled_wikis
project_id_matcher =
Geo::Fdw::ProjectFeature.arel_table[:project_id]
.eq(Geo::ProjectRegistry.arel_table[:project_id])
# Only read the IDs of projects with disabled wikis from the remote database
Geo::Fdw::ProjectFeature
.where(wiki_access_level: [::ProjectFeature::DISABLED])
.where(project_id_matcher)
.select('1')
.exists
.not
end
#
# Legacy accessors (non FDW)
#
......@@ -290,7 +272,7 @@ module Geo
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
def legacy_find_synced_wikis
legacy_inner_join_registry_ids(
current_node.projects.with_wiki_enabled,
current_node.projects,
Geo::ProjectRegistry.synced_wikis.pluck(:project_id),
Project
)
......@@ -304,7 +286,7 @@ module Geo
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified wikis
def legacy_find_verified_wikis
legacy_inner_join_registry_ids(
current_node.projects.with_wiki_enabled,
current_node.projects,
Geo::ProjectRegistry.verified_wikis.pluck(:project_id),
Project
)
......
......@@ -165,7 +165,7 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
end
def wiki_sync_due?(scheduled_time)
project.wiki_enabled? && (never_synced_wiki? || wiki_sync_needed?(scheduled_time))
never_synced_wiki? || wiki_sync_needed?(scheduled_time)
end
# Returns whether repository is pending verification check
......
......@@ -154,8 +154,7 @@ class GeoNodeStatus < ActiveRecord::Base
latest_event = Geo::EventLog.latest_event
self.last_event_id = latest_event&.id
self.last_event_date = latest_event&.created_at
self.repositories_count = projects_finder.count_repositories
self.wikis_count = projects_finder.count_wikis
self.projects_count = projects_finder.count_projects
self.lfs_objects_count = lfs_objects_finder.count_syncable
self.job_artifacts_count = job_artifacts_finder.count_syncable
self.attachments_count = attachments_finder.count_syncable
......@@ -163,6 +162,11 @@ class GeoNodeStatus < ActiveRecord::Base
self.storage_shards = StorageShard.all
self.storage_configuration_digest = StorageShard.build_digest
# Backward compatibility. These are deprecated and not used normally. Only needed for smooth update
# when secondary node is outdated yet
self.repositories_count = projects_finder.count_projects
self.wikis_count = projects_finder.count_projects
self.version = Gitlab::VERSION
self.revision = Gitlab.revision
......@@ -305,7 +309,7 @@ class GeoNodeStatus < ActiveRecord::Base
end
def repositories_synced_in_percentage
calc_percentage(repositories_count, repositories_synced_count)
calc_percentage(projects_count, repositories_synced_count)
end
def wikis_synced_in_percentage
......@@ -313,7 +317,7 @@ class GeoNodeStatus < ActiveRecord::Base
end
def repositories_checksummed_in_percentage
calc_percentage(repositories_count, repositories_checksummed_count)
calc_percentage(projects_count, repositories_checksummed_count)
end
def wikis_checksummed_in_percentage
......@@ -321,7 +325,7 @@ class GeoNodeStatus < ActiveRecord::Base
end
def repositories_verified_in_percentage
calc_percentage(repositories_count, repositories_verified_count)
calc_percentage(projects_count, repositories_verified_count)
end
def wikis_verified_in_percentage
......@@ -329,7 +333,7 @@ class GeoNodeStatus < ActiveRecord::Base
end
def repositories_checked_in_percentage
calc_percentage(repositories_count, repositories_checked_count)
calc_percentage(projects_count, repositories_checked_count)
end
def lfs_objects_synced_in_percentage
......@@ -359,6 +363,15 @@ class GeoNodeStatus < ActiveRecord::Base
public_send(key) # rubocop:disable GitlabSecurity/PublicSend
end
# This method is for backward compatibility only.
# During the app update the secondary node can be outdated, and it does not provide
# us with the projects_count which is already expected by updated primary node.
# So we just fallback to old repositories_count
def projects_count
projects_count_attr = read_attribute(:projects_count)
projects_count_attr.nil? ? repositories_count : projects_count_attr
end
private
def primary_storage_digest
......
......@@ -27,31 +27,8 @@ module Geo
return
end
mark_disabled_wiki_as_synced(registry)
Geo::RepositorySyncService.new(project).execute if registry.repository_sync_due?(scheduled_time)
Geo::WikiSyncService.new(project).execute if registry.wiki_sync_due?(scheduled_time)
end
private
def mark_disabled_wiki_as_synced(registry)
return if registry.project.wiki_enabled?
registry.last_wiki_sync_failure = nil
registry.resync_wiki = false
registry.wiki_retry_count = nil
registry.wiki_retry_at = nil
registry.force_to_redownload_wiki = false
if registry.changed? || registry.last_wiki_synced_at.nil? || registry.last_wiki_successful_sync_at.nil?
now = DateTime.now
registry.last_wiki_synced_at = now
registry.last_wiki_successful_sync_at = now
success = registry.save
log_info("#{success ? 'Successfully marked' : 'Failed to mark'} disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
end
end
end
end
---
title: 'Geo: sync disabled wikis'
merge_request: 6420
author:
type: fixed
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddProjectsCountToGeoNodeStatuses < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :geo_node_statuses, :projects_count, :integer
end
end
......@@ -315,14 +315,16 @@ module EE
number_to_percentage(node.job_artifacts_synced_in_percentage, precision: 2)
end
expose :repositories_count
expose :projects_count
expose :repositories_count # Deprecated
expose :repositories_failed_count
expose :repositories_synced_count
expose :repositories_synced_in_percentage do |node|
number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
end
expose :wikis_count
expose :wikis_count # Deprecated
expose :wikis_failed_count
expose :wikis_synced_count
expose :wikis_synced_in_percentage do |node|
......
......@@ -249,13 +249,13 @@ namespace :geo do
print 'Repositories: '.rjust(COLUMN_WIDTH)
show_failed_value(current_node_status.repositories_failed_count)
print "#{current_node_status.repositories_synced_count}/#{current_node_status.repositories_count} "
print "#{current_node_status.repositories_synced_count}/#{current_node_status.projects_count} "
puts using_percentage(current_node_status.repositories_synced_in_percentage)
if Gitlab::Geo.repository_verification_enabled?
print 'Verified Repositories: '.rjust(COLUMN_WIDTH)
show_failed_value(current_node_status.repositories_verification_failed_count)
print "#{current_node_status.repositories_verified_count}/#{current_node_status.repositories_count} "
print "#{current_node_status.repositories_verified_count}/#{current_node_status.projects_count} "
puts using_percentage(current_node_status.repositories_verified_in_percentage)
end
......@@ -284,7 +284,7 @@ namespace :geo do
if Gitlab::CurrentSettings.repository_checks_enabled
print 'Repositories Checked: '.rjust(COLUMN_WIDTH)
show_failed_value(current_node_status.repositories_checked_failed_count)
print "#{current_node_status.repositories_checked_count}/#{current_node_status.repositories_count} "
print "#{current_node_status.repositories_checked_count}/#{current_node_status.projects_count} "
puts using_percentage(current_node_status.repositories_checked_in_percentage)
end
......
......@@ -18,6 +18,7 @@ FactoryBot.define do
job_artifacts_synced_count 577
job_artifacts_synced_missing_on_primary_count 91
repositories_count 10
projects_count 10
repositories_synced_count 5
repositories_failed_count 0
wikis_count 9
......
......@@ -97,13 +97,6 @@ describe Geo::ProjectRegistryFinder, :geo do
expect(subject.count_synced_wikis).to eq 1
end
it 'does not count disabled wikis' do
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, project: create(:project, :wiki_disabled))
expect(subject.count_synced_wikis).to eq 1
end
context 'with selective sync' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
......@@ -248,13 +241,6 @@ describe Geo::ProjectRegistryFinder, :geo do
expect(subject.count_verified_wikis).to eq 2
end
it 'does not count disabled wikis' do
create(:geo_project_registry, :wiki_verified, project: project_wiki_verified)
create(:geo_project_registry, :wiki_verified, project: create(:project, :wiki_disabled))
expect(subject.count_verified_wikis).to eq 1
end
end
describe '#count_verification_failed_repositories' do
......
......@@ -20,6 +20,7 @@
"job_artifacts_synced_missing_on_primary_count",
"db_replication_lag_seconds",
"repositories_count",
"projects_count",
"repositories_failed_count",
"repositories_synced_count",
"wikis_count",
......@@ -83,6 +84,7 @@
"job_artifacts_synced_missing_on_primary_count": { "type": ["integer", "null"] },
"job_artifacts_synced_in_percentage": { "type": "string" },
"repositories_count": { "type": "integer" },
"projects_count": { "type": "integer" },
"repositories_failed_count": { "type": ["integer", "null"] },
"repository_verification_enabled": { "type": "boolean" },
"repositories_synced_count": { "type": ["integer", "null"] },
......
......@@ -85,7 +85,7 @@ describe EE::API::Entities::GeoNodeStatus, :postgresql do
describe '#repositories_synced_in_percentage' do
it 'formats as percentage' do
geo_node_status.assign_attributes(repositories_count: 10,
geo_node_status.assign_attributes(projects_count: 10,
repositories_synced_count: 5,
repositories_failed_count: 0)
......
......@@ -207,17 +207,7 @@ describe Geo::ProjectRegistry do
)
end
context 'wiki enabled' do
it { expect(registry.wiki_sync_due?(Time.now)).to eq(expected) }
end
context 'wiki disabled' do
before do
project.update!(wiki_enabled: false)
end
it { expect(registry.wiki_sync_due?(Time.now)).to be_falsy }
end
it { expect(registry.wiki_sync_due?(Time.now)).to eq(expected) }
end
end
......
......@@ -16,6 +16,7 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
status_message: nil,
db_replication_lag_seconds: 0,
repositories_count: 10,
projects_count: 10,
repositories_synced_count: 1,
repositories_failed_count: 2,
wikis_count: 10,
......@@ -56,6 +57,7 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
status_message: nil,
repositories_count: 10,
wikis_count: 10,
projects_count: 10,
lfs_objects_count: 100,
job_artifacts_count: 100,
attachments_count: 30,
......
......@@ -121,68 +121,6 @@ RSpec.describe Geo::ProjectSyncWorker do
end
end
context 'wiki is not enabled for project' do
let!(:registry) { create(:geo_project_registry, resync_repository: true, resync_wiki: true, project: project) }
before do
project.update!(wiki_enabled: false)
end
it 'syncs the project repository' do
subject.perform(project.id, Time.now)
expect(repository_sync_service).to have_received(:execute)
end
it 'does not sync the project wiki' do
subject.perform(project.id, Time.now)
expect(wiki_sync_service).not_to have_received(:execute)
end
context 'when the wiki has failed to sync before' do
let!(:registry) { create(:geo_project_registry, :wiki_sync_failed, project: project) }
it 'marks the wiki as synced, to remove it from failed Geo wiki queries' do
subject.perform(project.id, Time.now)
expect(registry.reload.resync_wiki).to be_falsey
expect(registry.reload.last_wiki_sync_failure).to be_nil
expect(registry.reload.last_wiki_synced_at).to be_present
expect(registry.reload.last_wiki_successful_sync_at).to be_present
expect(registry.reload.wiki_retry_count).to be_nil
expect(registry.reload.wiki_retry_at).to be_nil
expect(registry.reload.force_to_redownload_wiki).to be_falsey
end
it 'logs that the wiki was marked as not needing a sync' do
expect(subject).to receive(:log_info).with("Successfully marked disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
subject.perform(project.id, Time.now)
end
end
context 'when the wiki has never been synced before' do
it 'marks the wiki as synced, to remove it from out-of-sync Geo wiki queries' do
subject.perform(project.id, Time.now)
expect(registry.reload.resync_wiki).to be_falsey
expect(registry.reload.last_wiki_sync_failure).to be_nil
expect(registry.reload.last_wiki_synced_at).to be_present
expect(registry.reload.last_wiki_successful_sync_at).to be_present
expect(registry.reload.wiki_retry_count).to be_nil
expect(registry.reload.wiki_retry_at).to be_nil
expect(registry.reload.force_to_redownload_wiki).to be_falsey
end
it 'logs that the wiki was marked as not needing a sync' do
expect(subject).to receive(:log_info).with("Successfully marked disabled wiki as synced", registry_id: registry.id, project_id: registry.project_id)
subject.perform(project.id, Time.now)
end
end
end
context 'when project repository was synced after the time the job was scheduled in' do
it 'does not perform Geo::RepositorySyncService for the given project' do
create(:geo_project_registry, :synced, :repository_dirty, project: project, last_repository_synced_at: Time.now)
......
......@@ -37,6 +37,7 @@ module QA
db_replication_lag_seconds: :integer_or_null,
lfs_objects_count: :integer,
job_artifacts_count: :integer,
projects_count: :integer,
repositories_count: :integer,
wikis_count: :integer,
replication_slots_count: :integer_or_null,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment