Commit 025f297e authored by Nick Thomas's avatar Nick Thomas

Merge branch 'feature/geo-split-fail-repos' into 'master'

Split project repository and wiki repository status in Geo node status

Closes #3735

See merge request gitlab-org/gitlab-ee!3560
parents 2bd8ac34 931fd3a6
......@@ -25,6 +25,7 @@ class GeoNodeStatus {
this.$healthStatus = $('.js-health-status', this.$el);
this.$status = $('.js-geo-node-status', this.$el);
this.$repositories = $('.js-repositories', this.$status);
this.$wikis = $('.js-wikis', this.$status);
this.$lfsObjects = $('.js-lfs-objects', this.$status);
this.$attachments = $('.js-attachments', this.$status);
this.$syncSettings = $('.js-sync-settings', this.$status);
......@@ -219,6 +220,15 @@ class GeoNodeStatus {
GeoNodeStatus.renderSyncGraph(this.$repositories, repositoriesStats);
}
if (status.wikis_count > 0) {
const wikisStats = GeoNodeStatus.getSyncStatistics({
syncedCount: status.wikis_synced_count,
failedCount: status.wikis_failed_count,
totalCount: status.wikis_count,
});
GeoNodeStatus.renderSyncGraph(this.$wikis, wikisStats);
}
if (status.lfs_objects_count > 0) {
const lfsObjectsStats = GeoNodeStatus.getSyncStatistics({
syncedCount: status.lfs_objects_synced_count,
......
---
title: Split project repository and wiki repository status in Geo node status
merge_request: 3560
author:
type: added
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddWikiInfoToGeoNodeStatuses < ActiveRecord::Migration
DOWNTIME = false
def change
add_column :geo_node_statuses, :wikis_count, :integer
add_column :geo_node_statuses, :wikis_synced_count, :integer
add_column :geo_node_statuses, :wikis_failed_count, :integer
end
end
......@@ -970,6 +970,9 @@ ActiveRecord::Schema.define(version: 20171213160445) do
t.datetime "updated_at", null: false
t.datetime "last_successful_status_check_at"
t.string "status_message"
t.integer "wikis_count"
t.integer "wikis_synced_count"
t.integer "wikis_failed_count"
end
add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree
......
......@@ -93,6 +93,10 @@ Example response:
"repositories_failed_count": 1,
"repositories_synced_count": 40,
"repositories_synced_in_percentage": "97.56%",
"wikis_count": 41,
"wikis_failed_count": 0,
"wikis_synced_count": 41,
"wikis_synced_in_percentage": "100.00%",
"last_event_id": 23,
"last_event_timestamp": 1509681166,
"cursor_last_event_id": 23,
......
module Geo
class ProjectRegistryFinder < RegistryFinder
def count_projects
def count_repositories
current_node.projects.count
end
def count_synced_project_registries
def count_wikis
current_node.projects.with_wiki_enabled.count
end
def count_synced_repositories
relation =
if selective_sync?
legacy_find_synced_projects
legacy_find_synced_repositories
else
find_synced_project_registries
find_synced_repositories
end
relation.count
end
def count_failed_project_registries
find_failed_project_registries.count
def count_synced_wikis
relation =
if use_legacy_queries?
legacy_find_synced_wikis
else
fdw_find_enabled_wikis
end
relation.count
end
def count_failed_repositories
find_failed_project_registries('repository').count
end
def count_failed_wikis
find_failed_project_registries('wiki').count
end
def find_failed_project_registries(type = nil)
......@@ -54,8 +73,8 @@ module Geo
protected
def find_synced_project_registries
Geo::ProjectRegistry.synced
def find_synced_repositories
Geo::ProjectRegistry.synced_repos
end
def find_filtered_failed_project_registries(type = nil)
......@@ -83,11 +102,20 @@ module Geo
.where(project_registry: { project_id: nil })
end
# @return [ActiveRecord::Relation<Geo::Fdw::Project>]
def fdw_find_enabled_wikis
feature_fdw_table = Geo::Fdw::ProjectFeature.table_name
Geo::ProjectRegistry.synced_wikis
.joins("INNER JOIN #{feature_fdw_table} ON #{feature_fdw_table}.project_id = project_registry.project_id")
.where("#{feature_fdw_table}.wiki_access_level > ?", ::ProjectFeature::DISABLED)
end
# @return [ActiveRecord::Relation<Geo::Fdw::Project>]
def fdw_find_projects_updated_recently
Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
.merge(Geo::ProjectRegistry.dirty)
.merge(Geo::ProjectRegistry.retry_due)
.merge(Geo::ProjectRegistry.dirty)
.merge(Geo::ProjectRegistry.retry_due)
end
#
......@@ -112,11 +140,25 @@ module Geo
)
end
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
def legacy_find_synced_repositories
legacy_find_project_registries(Geo::ProjectRegistry.synced_repos)
end
# @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
def legacy_find_synced_wikis
legacy_inner_join_registry_ids(
current_node.projects.with_wiki_enabled,
Geo::ProjectRegistry.synced_wikis.pluck(:project_id),
Project
)
end
# @return [ActiveRecord::Relation<Project>] list of synced projects
def legacy_find_synced_projects
def legacy_find_project_registries(project_registries)
legacy_inner_join_registry_ids(
current_node.projects,
Geo::ProjectRegistry.synced.pluck(:project_id),
project_registries.pluck(:project_id),
Project
)
end
......
module Geo
module Fdw
class ProjectFeature < ::Geo::BaseFdw
self.table_name = Gitlab::Geo.fdw_table('project_features')
end
end
end
......@@ -23,9 +23,14 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
)
end
def self.synced
def self.synced_repos
where.not(last_repository_synced_at: nil, last_repository_successful_sync_at: nil)
.where(resync_repository: false, resync_wiki: false)
.where(resync_repository: false)
end
def self.synced_wikis
where.not(last_wiki_synced_at: nil, last_wiki_successful_sync_at: nil)
.where(resync_wiki: false)
end
def repository_sync_due?(scheduled_time)
......
......@@ -62,7 +62,8 @@ class GeoNodeStatus < ActiveRecord::Base
latest_event = Geo::EventLog.latest_event
self.last_event_id = latest_event&.id
self.last_event_date = latest_event&.created_at
self.repositories_count = projects_finder.count_projects
self.repositories_count = projects_finder.count_repositories
self.wikis_count = projects_finder.count_wikis
self.lfs_objects_count = lfs_objects_finder.count_lfs_objects
self.attachments_count = attachments_finder.count_attachments
self.last_successful_status_check_at = Time.now
......@@ -71,8 +72,10 @@ class GeoNodeStatus < ActiveRecord::Base
self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.db_replication_lag_seconds
self.cursor_last_event_id = Geo::EventLogState.last_processed&.event_id
self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
self.repositories_synced_count = projects_finder.count_synced_project_registries
self.repositories_failed_count = projects_finder.count_failed_project_registries
self.repositories_synced_count = projects_finder.count_synced_repositories
self.repositories_failed_count = projects_finder.count_failed_repositories
self.wikis_synced_count = projects_finder.count_synced_wikis
self.wikis_failed_count = projects_finder.count_failed_wikis
self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
self.attachments_synced_count = attachments_finder.count_synced_attachments
......@@ -120,6 +123,10 @@ class GeoNodeStatus < ActiveRecord::Base
sync_percentage(repositories_count, repositories_synced_count)
end
def wikis_synced_in_percentage
sync_percentage(wikis_count, wikis_synced_count)
end
def lfs_objects_synced_in_percentage
sync_percentage(lfs_objects_count, lfs_objects_synced_count)
end
......
......@@ -32,6 +32,13 @@ class GeoNodeStatusEntity < Grape::Entity
number_to_percentage(node.repositories_synced_in_percentage, precision: 2)
end
expose :wikis_count
expose :wikis_failed_count
expose :wikis_synced_count
expose :wikis_synced_in_percentage do |node|
number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
end
expose :last_event_id
expose :last_event_timestamp
expose :cursor_last_event_id
......
......@@ -64,6 +64,17 @@
%span.status-green.js-synced
%span.status-neutral.js-waiting
%span.status-red.js-failed
%tr
%td
.help-block.prepend-top-10
Wikis:
%td
.node-info.prepend-top-10.prepend-left-5.stacked-progress-bar.js-wikis
%span.status-unavailable.js-stats-unavailable
Not available
%span.status-green.js-synced
%span.status-neutral.js-waiting
%span.status-red.js-failed
%tr
%td
.help-block.prepend-top-10
......
......@@ -16,20 +16,20 @@ describe Geo::ProjectRegistryFinder, :geo do
stub_current_geo_node(secondary)
end
describe '#count_synced_project_registries' do
it 'delegates to #find_synced_project_registries' do
expect(subject).to receive(:find_synced_project_registries).and_call_original
describe '#count_synced_repositories' do
it 'delegates to #find_synced_repositories' do
expect(subject).to receive(:find_synced_repositories).and_call_original
subject.count_synced_project_registries
subject.count_synced_repositories
end
it 'counts projects that has been synced' do
it 'counts repositories that have been synced' do
create(:geo_project_registry, :sync_failed)
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
expect(subject.count_synced_project_registries).to eq 1
expect(subject.count_synced_repositories).to eq 2
end
context 'with selective sync' do
......@@ -37,10 +37,10 @@ describe Geo::ProjectRegistryFinder, :geo do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #legacy_find_synced_projects' do
expect(subject).to receive(:legacy_find_synced_projects).and_call_original
it 'delegates to #legacy_find_synced_repositories' do
expect(subject).to receive(:legacy_find_synced_repositories).and_call_original
subject.count_synced_project_registries
subject.count_synced_repositories
end
it 'counts projects that has been synced' do
......@@ -51,16 +51,67 @@ describe Geo::ProjectRegistryFinder, :geo do
create(:geo_project_registry, :synced, project: project_1_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
expect(subject.count_synced_project_registries).to eq 1
expect(subject.count_synced_repositories).to eq 1
end
end
end
describe '#count_failed_project_registries' do
describe '#count_synced_wikis' do
before do
allow(subject).to receive(:use_legacy_queries?).and_return(true)
end
it 'delegates to #legacy_find_synced_wikis' do
expect(subject).to receive(:legacy_find_synced_wikis).and_call_original
subject.count_synced_wikis
end
it 'counts wiki that have been synced' do
create(:geo_project_registry, :sync_failed)
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
expect(subject.count_synced_wikis).to eq 2
end
it 'does not count disabled wikis' do
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, project: create(:project, :wiki_disabled))
expect(subject.count_synced_wikis).to eq 1
end
context 'with selective sync' do
before do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #legacy_find_synced_wiki' do
expect(subject).to receive(:legacy_find_synced_wikis).and_call_original
subject.count_synced_wikis
end
it 'counts projects that has been synced' do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, project: project_1_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
expect(subject.count_synced_wikis).to eq 1
end
end
end
describe '#count_failed_repositories' do
it 'delegates to #find_failed_project_registries' do
expect(subject).to receive(:find_failed_project_registries).and_call_original
expect(subject).to receive(:find_failed_project_registries).with('repository').and_call_original
subject.count_failed_project_registries
subject.count_failed_repositories
end
it 'counts projects that sync has failed' do
......@@ -69,7 +120,7 @@ describe Geo::ProjectRegistryFinder, :geo do
create(:geo_project_registry, :repository_sync_failed, project: project_repository_dirty)
create(:geo_project_registry, :wiki_sync_failed, project: project_wiki_dirty)
expect(subject.count_failed_project_registries).to eq 3
expect(subject.count_failed_repositories).to eq 2
end
context 'with selective sync' do
......@@ -77,10 +128,10 @@ describe Geo::ProjectRegistryFinder, :geo do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #find_failed_project_registries' do
expect(subject).to receive(:find_failed_project_registries).and_call_original
it 'delegates to #find_failed_repositories' do
expect(subject).to receive(:find_failed_project_registries).with('repository').and_call_original
subject.count_failed_project_registries
subject.count_failed_repositories
end
it 'counts projects that sync has failed' do
......@@ -91,7 +142,47 @@ describe Geo::ProjectRegistryFinder, :geo do
create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, project: project_2_in_synced_group)
expect(subject.count_failed_project_registries).to eq 1
expect(subject.count_failed_repositories).to eq 1
end
end
end
describe '#count_failed_wikis' do
it 'delegates to #find_failed_project_registries' do
expect(subject).to receive(:find_failed_project_registries).with('wiki').and_call_original
subject.count_failed_wikis
end
it 'counts projects that sync has failed' do
create(:geo_project_registry, :synced)
create(:geo_project_registry, :sync_failed, project: project_synced)
create(:geo_project_registry, :repository_sync_failed, project: project_repository_dirty)
create(:geo_project_registry, :wiki_sync_failed, project: project_wiki_dirty)
expect(subject.count_failed_wikis).to eq 2
end
context 'with selective sync' do
before do
secondary.update_attribute(:namespaces, [synced_group])
end
it 'delegates to #find_failed_wikis' do
expect(subject).to receive(:find_failed_project_registries).with('wiki').and_call_original
subject.count_failed_wikis
end
it 'counts projects that sync has failed' do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :sync_failed, project: project_synced)
create(:geo_project_registry, :wiki_sync_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, project: project_2_in_synced_group)
expect(subject.count_failed_wikis).to eq 1
end
end
end
......@@ -105,14 +196,10 @@ describe Geo::ProjectRegistryFinder, :geo do
let!(:repository_sync_failed) { create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group) }
let!(:wiki_sync_failed) { create(:geo_project_registry, :wiki_sync_failed, project: project_2_in_synced_group) }
it 'delegates to #find_filtered_failed_project_registries' do
expect(subject).to receive(:find_filtered_failed_project_registries).and_call_original
subject.find_failed_project_registries
end
it 'delegates to #find_failed_project_registries' do
expect(subject).to receive(:find_failed_project_registries).with('repository').and_call_original
it 'returns project registries that sync has failed' do
expect(subject.find_failed_project_registries).to match_array([sync_failed, repository_sync_failed, wiki_sync_failed])
subject.count_failed_repositories
end
it 'returns only project registries that repository sync has failed' do
......@@ -159,6 +246,17 @@ describe Geo::ProjectRegistryFinder, :geo do
skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo.fdw?
end
describe '#fdw_find_enabled_wikis' do
it 'does not count disabled wikis' do
expect(subject).to receive(:fdw_find_enabled_wikis).and_call_original
create(:geo_project_registry, :synced, project: project_synced)
create(:geo_project_registry, :synced, project: create(:project, :wiki_disabled))
expect(subject.count_synced_wikis).to eq 1
end
end
describe '#find_unsynced_projects' do
it 'delegates to #fdw_find_unsynced_projects' do
expect(subject).to receive(:fdw_find_unsynced_projects).and_call_original
......
......@@ -17,27 +17,31 @@ describe Geo::ProjectRegistry do
it { is_expected.to validate_uniqueness_of(:project) }
end
describe '.failed' do
describe '.failed_repos' do
it 'returns projects where last attempt to sync failed' do
create(:geo_project_registry, :synced)
create(:geo_project_registry, :synced, :dirty)
create(:geo_project_registry, :repository_syncing)
create(:geo_project_registry, :wiki_syncing)
create(:geo_project_registry, :wiki_sync_failed)
repository_sync_failed = create(:geo_project_registry, :repository_sync_failed)
wiki_sync_failed = create(:geo_project_registry, :wiki_sync_failed)
expect(described_class.failed).to match_array([repository_sync_failed, wiki_sync_failed])
expect(described_class.failed_repos).to match_array([repository_sync_failed])
end
end
describe '.synced' do
it 'returns synced projects' do
describe '.failed_wikis' do
it 'returns projects where last attempt to sync failed' do
create(:geo_project_registry, :synced)
create(:geo_project_registry, :synced, :dirty)
create(:geo_project_registry, :sync_failed)
synced_project = create(:geo_project_registry, :synced)
create(:geo_project_registry, :repository_syncing)
create(:geo_project_registry, :wiki_syncing)
create(:geo_project_registry, :repository_sync_failed)
wiki_sync_failed = create(:geo_project_registry, :wiki_sync_failed)
expect(described_class.synced).to match_array([synced_project])
expect(described_class.failed_wikis).to match_array([wiki_sync_failed])
end
end
......
......@@ -217,6 +217,25 @@ describe GeoNodeStatus, :geo do
end
end
describe '#wikis_failed_count' do
before do
create(:geo_project_registry, :sync_failed, project: project_1)
create(:geo_project_registry, :sync_failed, project: project_3)
create(:geo_project_registry, :repository_syncing, project: project_4)
create(:geo_project_registry, :wiki_syncing)
end
it 'returns the right number of failed repos with no group restrictions' do
expect(subject.wikis_failed_count).to eq(2)
end
it 'returns the right number of failed repos with group restrictions' do
secondary.update_attribute(:namespaces, [group])
expect(subject.wikis_failed_count).to eq(1)
end
end
describe '#repositories_synced_in_percentage' do
it 'returns 0 when no projects are available' do
expect(subject.repositories_synced_in_percentage).to eq(0)
......@@ -242,6 +261,33 @@ describe GeoNodeStatus, :geo do
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
describe '#wikis_synced_in_percentage', :delete do
it 'returns 0 when no projects are available' do
expect(subject.wikis_synced_in_percentage).to eq(0)
end
it 'returns 0 when project count is unknown' do
allow(subject).to receive(:wikis_count).and_return(nil)
expect(subject.wikis_synced_in_percentage).to eq(0)
end
it 'returns the right percentage with no group restrictions' do
create(:geo_project_registry, :synced, project: project_1)
expect(subject.wikis_synced_in_percentage).to be_within(0.0001).of(25)
end
it 'returns the right percentage with group restrictions' do
secondary.update_attribute(:namespaces, [group])
create(:geo_project_registry, :synced, project: project_1)
expect(subject.wikis_synced_in_percentage).to be_within(0.0001).of(50)
end
end
describe '#last_event_id and #last_event_date' do
it 'returns nil when no events are available' do
expect(subject.last_event_id).to be_nil
......
......@@ -22,6 +22,10 @@ describe GeoNodeStatusEntity, :postgresql do
it { is_expected.to have_key(:repositories_failed_count) }
it { is_expected.to have_key(:repositories_synced_count)}
it { is_expected.to have_key(:repositories_synced_in_percentage) }
it { is_expected.to have_key(:wikis_count) }
it { is_expected.to have_key(:wikis_failed_count) }
it { is_expected.to have_key(:wikis_synced_count)}
it { is_expected.to have_key(:wikis_synced_in_percentage) }
it { is_expected.to have_key(:last_successful_status_check_timestamp) }
it { is_expected.to have_key(:namespaces) }
......
......@@ -14,6 +14,9 @@ FactoryBot.define do
repositories_count 10
repositories_synced_count 5
repositories_failed_count 0
wikis_count 9
wikis_synced_count 4
wikis_failed_count 1
last_event_id 2
last_event_timestamp Time.now.to_i
cursor_last_event_id 1
......
......@@ -15,6 +15,9 @@
"repositories_count",
"repositories_failed_count",
"repositories_synced_count",
"wikis_count",
"wikis_failed_count",
"wikis_synced_count",
"last_event_id",
"last_event_timestamp",
"cursor_last_event_id",
......@@ -41,6 +44,10 @@
"repositories_failed_count": { "type": "integer" },
"repositories_synced_count": { "type": "integer" },
"repositories_synced_in_percentage": { "type": "string" },
"wikis_count": { "type": "integer" },
"wikis_failed_count": { "type": "integer" },
"wikis_synced_count": { "type": "integer" },
"wikis_synced_in_percentage": { "type": "string" },
"last_event_id": { "type": ["integer", "null"] },
"last_event_timestamp": { "type": ["integer", "null"] },
"cursor_last_event_id": { "type": ["integer", "null"] },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment