Commit 0122ca3b authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch '13055-show-the-sync-information-for-design-repositories' into 'master'

Show the sync information for design repositories

Closes #13055

See merge request gitlab-org/gitlab!18379
parents 17a7203c d4b17960
---
title: 'Geo: Add resigns-related fields to Geo Node Status table'
merge_request: 18379
author:
type: other
# frozen_string_literal: true
class AddGeoDesignRepositoryCounters < ActiveRecord::Migration[5.1]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
change_table :geo_node_statuses do |t|
t.column :design_repositories_count, :integer
t.column :design_repositories_synced_count, :integer
t.column :design_repositories_failed_count, :integer
t.column :design_repositories_registry_count, :integer
end
end
end
......@@ -1650,6 +1650,10 @@ ActiveRecord::Schema.define(version: 2019_10_17_045817) do
t.integer "container_repositories_synced_count"
t.integer "container_repositories_failed_count"
t.integer "container_repositories_registry_count"
t.integer "design_repositories_count"
t.integer "design_repositories_synced_count"
t.integer "design_repositories_failed_count"
t.integer "design_repositories_registry_count"
t.index ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true
end
......
......@@ -237,6 +237,10 @@ Example response:
"container_repositories_synced_count": nil,
"container_repositories_failed_count": nil,
"container_repositories_synced_in_percentage": "0.00%",
"design_repositories_count": 3,
"design_repositories_synced_count": nil,
"design_repositories_failed_count": nil,
"design_repositories_synced_in_percentage": "0.00%",
"projects_count": 41,
"repositories_failed_count": nil,
"repositories_synced_count": nil,
......@@ -304,6 +308,10 @@ Example response:
"container_repositories_synced_count": nil,
"container_repositories_failed_count": nil,
"container_repositories_synced_in_percentage": "0.00%",
"design_repositories_count": 3,
"design_repositories_synced_count": nil,
"design_repositories_failed_count": nil,
"design_repositories_synced_in_percentage": "0.00%",
"projects_count": 41,
"repositories_failed_count": 1,
"repositories_synced_count": 40,
......@@ -387,6 +395,10 @@ Example response:
"container_repositories_synced_count": nil,
"container_repositories_failed_count": nil,
"container_repositories_synced_in_percentage": "0.00%",
"design_repositories_count": 3,
"design_repositories_synced_count": nil,
"design_repositories_failed_count": nil,
"design_repositories_synced_in_percentage": "0.00%",
"projects_count": 41,
"repositories_failed_count": 1,
"repositories_synced_count": 40,
......
......@@ -79,6 +79,11 @@ export default {
required: false,
default: false,
},
featureDisabled: {
type: Boolean,
required: false,
default: false,
},
},
computed: {
hasHelpInfo() {
......@@ -121,9 +126,9 @@ export default {
</script>
<template>
<div class="prepend-top-15 prepend-left-10 node-detail-item">
<div v-if="!featureDisabled" class="prepend-top-15 prepend-left-10 node-detail-item">
<div class="node-detail-title">
<span> {{ itemTitle }} </span>
<span>{{ itemTitle }}</span>
<icon
v-if="hasHelpInfo"
v-popover="popoverConfig"
......
......@@ -61,6 +61,12 @@ export default {
itemValue: this.nodeDetails.containerRepositories,
itemValueType: VALUE_TYPE.GRAPH,
},
{
itemTitle: s__('GeoNodes|Design repositories'),
itemValue: this.nodeDetails.designRepositories,
itemValueType: VALUE_TYPE.GRAPH,
featureDisabled: !gon.features.enableGeoDesignSync,
},
{
itemTitle: s__('GeoNodes|Data replication lag'),
itemValue: this.dbReplicationLag(),
......@@ -146,6 +152,7 @@ export default {
:item-value-stale-tooltip="statusInfoStaleMessage"
:custom-type="nodeDetailItem.customType"
:event-type-log-status="nodeDetailItem.eventTypeLogStatus"
:feature-disabled="nodeDetailItem.featureDisabled"
/>
</div>
</div>
......
......@@ -124,6 +124,11 @@ export default class GeoNodesStore {
successCount: rawNodeDetails.container_repositories_synced_count || 0,
failureCount: rawNodeDetails.container_repositories_failed_count || 0,
},
designRepositories: {
totalCount: rawNodeDetails.design_repositories_count || 0,
successCount: rawNodeDetails.design_repositories_synced_count || 0,
failureCount: rawNodeDetails.design_repositories_failed_count || 0,
},
attachments: {
totalCount: rawNodeDetails.attachments_count || 0,
successCount: rawNodeDetails.attachments_synced_count || 0,
......
......@@ -3,6 +3,9 @@
class Admin::Geo::NodesController < Admin::Geo::ApplicationController
before_action :check_license!, except: :index
before_action :load_node, only: [:edit, :update]
before_action only: [:index] do
push_frontend_feature_flag(:enable_geo_design_sync)
end
# rubocop: disable CodeReuse/ActiveRecord
def index
......
# frozen_string_literal: true
module Geo
class DesignRegistryFinder < RegistryFinder
def count_syncable
designs_repositories.count
end
def count_synced
registries_for_design_repositories
.merge(Geo::DesignRegistry.synced).count
end
def count_failed
registries_for_design_repositories
.merge(Geo::DesignRegistry.failed).count
end
def count_registry
registries_for_design_repositories.count
end
private
def designs_repositories
current_node.projects.inner_join_design_management
end
def registries_for_design_repositories
designs_repositories
.inner_join_design_registry
end
end
end
# frozen_string_literal: true
module Geo
module Fdw
class DesignManagementDesign < ::Geo::BaseFdw
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('design_management_designs')
self.primary_key = :id
end
end
end
......@@ -15,6 +15,7 @@ module Geo
has_many :container_repositories, class_name: 'Geo::Fdw::ContainerRepository'
belongs_to :namespace, class_name: 'Geo::Fdw::Namespace'
belongs_to :design_management_designs, class_name: 'Geo::Fdw::DesignManagementDesign'
scope :outside_shards, -> (shard_names) { where.not(repository_storage: Array(shard_names)) }
......@@ -80,6 +81,24 @@ module Geo
joins(join_statement.join_sources)
end
def inner_join_design_registry
join_statement =
arel_table
.join(Geo::DesignRegistry.arel_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(Geo::DesignRegistry.arel_table[:project_id]))
joins(join_statement.join_sources)
end
def inner_join_design_management
join_statement =
arel_table
.join(Geo::Fdw::DesignManagementDesign.arel_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(Geo::Fdw::DesignManagementDesign.arel_table[:project_id]))
joins(join_statement.join_sources)
end
private
def left_outer_join_project_registry
......
......@@ -91,7 +91,11 @@ class GeoNodeStatus < ApplicationRecord
container_repositories_count: 'Total number of syncable container repositories available on primary',
container_repositories_synced_count: 'Number of syncable container repositories synced on secondary',
container_repositories_failed_count: 'Number of syncable container repositories failed to sync on secondary',
container_repositories_registry_count: 'Number of container repositories in the registry'
container_repositories_registry_count: 'Number of container repositories in the registry',
design_repositories_count: 'Total number of syncable design repositories available on primary',
design_repositories_synced_count: 'Number of syncable design repositories synced on secondary',
design_repositories_failed_count: 'Number of syncable design repositories failed to sync on secondary',
design_repositories_registry_count: 'Number of design repositories in the registry'
}.freeze
EXPIRATION_IN_MINUTES = 5
......@@ -240,6 +244,7 @@ class GeoNodeStatus < ApplicationRecord
attr_in_percentage :attachments_synced, :attachments_synced_count, :attachments_count
attr_in_percentage :replication_slots_used, :replication_slots_used_count, :replication_slots_count
attr_in_percentage :container_repositories_synced, :container_repositories_synced_count, :container_repositories_count
attr_in_percentage :design_repositories_synced, :design_repositories_synced_count, :design_repositories_count
def storage_shards_match?
return true if geo_node.primary?
......@@ -303,6 +308,7 @@ class GeoNodeStatus < ApplicationRecord
load_job_artifacts_data
load_attachments_data
load_container_registry_data
load_designs_data
end
def load_lfs_objects_data
......@@ -330,10 +336,17 @@ class GeoNodeStatus < ApplicationRecord
end
def load_container_registry_data
self.container_repositories_count = container_repository_finder.count_syncable
self.container_repositories_synced_count = container_repository_finder.count_synced
self.container_repositories_failed_count = container_repository_finder.count_failed
self.container_repositories_registry_count = container_repository_finder.count_registry
self.container_repositories_count = container_registry_finder.count_syncable
self.container_repositories_synced_count = container_registry_finder.count_synced
self.container_repositories_failed_count = container_registry_finder.count_failed
self.container_repositories_registry_count = container_registry_finder.count_registry
end
def load_designs_data
self.design_repositories_count = design_registry_finder.count_syncable
self.design_repositories_synced_count = design_registry_finder.count_synced
self.design_repositories_failed_count = design_registry_finder.count_failed
self.design_repositories_registry_count = design_registry_finder.count_registry
end
def load_repository_check_data
......@@ -382,8 +395,12 @@ class GeoNodeStatus < ApplicationRecord
@job_artifacts_finder ||= Geo::JobArtifactRegistryFinder.new(current_node_id: geo_node.id)
end
def container_repository_finder
@container_repository_finder ||= Geo::ContainerRepositoryRegistryFinder.new(current_node_id: geo_node.id)
def container_registry_finder
@container_registry_finder ||= Geo::ContainerRepositoryRegistryFinder.new(current_node_id: geo_node.id)
end
def design_registry_finder
@design_registry_finder ||= Geo::DesignRegistryFinder.new(current_node_id: geo_node.id)
end
def registries_for_synced_projects(type)
......
......@@ -649,6 +649,13 @@ module EE
number_to_percentage(node.container_repositories_synced_in_percentage, precision: 2)
end
expose :design_repositories_count
expose :design_repositories_synced_count
expose :design_repositories_failed_count
expose :design_repositories_synced_in_percentage do |node|
number_to_percentage(node.design_repositories_synced_in_percentage, precision: 2)
end
expose :projects_count
expose :repositories_failed_count
......
......@@ -339,6 +339,13 @@ namespace :geo do
puts using_percentage(current_node_status.container_repositories_synced_in_percentage)
end
if Feature.enabled?(:enable_geo_design_sync)
print 'Design repositories: '.rjust(GEO_STATUS_COLUMN_WIDTH)
show_failed_value(current_node_status.design_repositories_failed_count)
print "#{current_node_status.design_repositories_synced_count || 0}/#{current_node_status.design_repositories_count || 0} "
puts using_percentage(current_node_status.design_repositories_synced_in_percentage)
end
if Gitlab::CurrentSettings.repository_checks_enabled
print 'Repositories Checked: '.rjust(GEO_STATUS_COLUMN_WIDTH)
show_failed_value(current_node_status.repositories_checked_failed_count)
......
......@@ -7,6 +7,10 @@ FactoryBot.define do
last_synced_at { nil }
state { :pending }
after(:create) do |registry, evaluator|
create(:design, project: registry.project)
end
trait :synced do
state { :synced }
last_synced_at { 5.days.ago }
......
......@@ -22,6 +22,9 @@ FactoryBot.define do
container_repositories_count { 400 }
container_repositories_failed_count { 3 }
container_repositories_synced_count { 200 }
design_repositories_count { 400 }
design_repositories_failed_count { 3 }
design_repositories_synced_count { 200 }
projects_count { 10 }
repositories_synced_count { 5 }
repositories_failed_count { 0 }
......
# frozen_string_literal: true
require 'spec_helper'
describe Geo::DesignRegistryFinder, :geo, :geo_fdw do
include ::EE::GeoHelpers
let!(:secondary) { create(:geo_node) }
let!(:failed_registry) { create(:geo_design_registry, :sync_failed) }
let!(:synced_registry) { create(:geo_design_registry, :synced) }
let(:synced_group) { create(:group) }
let(:unsynced_group) { create(:group) }
let(:synced_project) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) }
subject { described_class.new(current_node_id: secondary.id) }
before do
stub_current_geo_node(secondary)
end
context 'count all the things' do
describe '#count_syncable' do
it 'returns number of container repositories' do
result = subject.count_syncable
expect(result).to eq(2)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
result = subject.count_synced
expect(result).to eq(1)
end
end
describe '#count_failed' do
it 'returns only failed registry' do
result = subject.count_failed
expect(result).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
result = subject.count_registry
expect(result).to eq(2)
end
end
context 'selective sync' do
let(:unsynced_project2) { create(:project, group: unsynced_group) }
let(:synced_project2) { create(:project, group: synced_group) }
before do
create(:geo_design_registry, :synced, project: synced_project)
create(:geo_design_registry, :sync_failed, project: synced_project2)
create(:geo_design_registry, :synced, project: unsynced_project)
create(:geo_design_registry, :sync_failed, project: unsynced_project2)
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
context 'count all the things' do
describe '#count_syncable' do
it 'returns number of container repositories' do
result = subject.count_syncable
expect(result).to eq(2)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
result = subject.count_synced
expect(result).to eq(1)
end
end
describe '#count_failed' do
it 'returns only failed registry' do
result = subject.count_failed
expect(result).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
result = subject.count_registry
expect(result).to eq(2)
end
end
end
end
end
end
......@@ -22,6 +22,9 @@
"container_repositories_count",
"container_repositories_failed_count",
"container_repositories_synced_count",
"design_repositories_count",
"design_repositories_failed_count",
"design_repositories_synced_count",
"projects_count",
"repositories_failed_count",
"repositories_synced_count",
......@@ -88,6 +91,10 @@
"container_repositories_failed_count": { "type": ["integer", "null"] },
"container_repositories_synced_count": { "type": ["integer", "null"] },
"container_repositories_synced_in_percentage": { "type": "string" },
"design_repositories_count": { "type": "integer" },
"design_repositories_failed_count": { "type": ["integer", "null"] },
"design_repositories_synced_count": { "type": ["integer", "null"] },
"design_repositories_synced_in_percentage": { "type": "string" },
"projects_count": { "type": "integer" },
"repositories_failed_count": { "type": ["integer", "null"] },
"repository_verification_enabled": { "type": "boolean" },
......
......@@ -117,5 +117,14 @@ describe('GeoNodeDetailItemComponent', () => {
expect(vm.$el.querySelectorAll('.event-status-timestamp').length).not.toBe(0);
vm.$destroy();
});
it('does not render if featureDisabled is true', () => {
const vm = createComponent({
featureDisabled: true,
});
expect(vm.$el.innerHTML).toBeUndefined();
vm.$destroy();
});
});
});
......@@ -16,6 +16,7 @@ describe('NodeDetailsSectionSync', () => {
let vm;
beforeEach(() => {
gon.features = gon.features || {};
vm = createComponent();
});
......
......@@ -83,6 +83,10 @@ export const rawMockNodeDetails = {
container_repositories_synced_count: 0,
container_repositories_failed_count: 0,
container_repositories_synced_in_percentage: '0.00%',
design_repositories_count: 0,
design_repositories_synced_count: 0,
design_repositories_failed_count: 0,
design_repositories_synced_in_percentage: '0.00%',
repositories_failed_count: 0,
repositories_synced_count: 12,
repositories_synced_in_percentage: '100.00%',
......@@ -193,6 +197,11 @@ export const mockNodeDetails = {
successCount: 0,
failureCount: 0,
},
designRepositories: {
totalCount: 0,
successCount: 0,
failureCount: 0,
},
attachments: {
totalCount: 0,
successCount: 0,
......
......@@ -97,6 +97,18 @@ describe EE::API::Entities::GeoNodeStatus do
end
end
describe '#design_repositories_synced_in_percentage' do
it 'formats as percentage' do
geo_node_status.assign_attributes(
design_repositories_count: 256,
design_repositories_failed_count: 12,
design_repositories_synced_count: 123
)
expect(subject[:design_repositories_synced_in_percentage]).to eq '48.05%'
end
end
describe '#repositories_synced_in_percentage' do
it 'formats as percentage' do
geo_node_status.assign_attributes(projects_count: 10,
......
......@@ -705,6 +705,67 @@ describe GeoNodeStatus, :geo, :geo_fdw do
end
end
describe '#design_repositories_count' do
it 'counts all the designs' do
create(:design)
create(:design)
expect(subject.design_repositories_count).to eq(2)
end
end
describe '#design_repositories_synced_count' do
it 'counts synced repositories' do
create(:geo_design_registry, :synced)
create(:geo_design_registry, :sync_failed)
expect(subject.design_repositories_synced_count).to eq(1)
end
end
describe '#design_repositories_failed_count' do
it 'counts failed to sync repositories' do
create(:geo_design_registry, :sync_failed)
create(:geo_design_registry, :synced)
expect(subject.design_repositories_failed_count).to eq(1)
end
end
describe '#design_repositories_registry_count' do
it 'counts number of registries for repositories' do
create(:geo_design_registry, :sync_failed)
create(:geo_design_registry)
create(:geo_design_registry, :synced)
expect(subject.design_repositories_registry_count).to eq(3)
end
end
describe '#design_repositories_synced_in_percentage' do
it 'returns 0 when no objects are available' do
expect(subject.design_repositories_synced_in_percentage).to eq(0)
end
it 'returns the right percentage with no group restrictions' do
create(:geo_design_registry, :synced)
create(:geo_design_registry, :sync_failed)
expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:geo_design_registry, :synced, project: project_1)
create(:geo_design_registry, :sync_failed, project: project_2)
create(:geo_design_registry, :sync_failed, project: project_3)
create(:geo_design_registry, :sync_failed, project: project_4)
expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
end
end
describe '#repositories_verified_count' do
before do
stub_current_geo_node(secondary)
......
......@@ -270,6 +270,9 @@ describe API::Geo do
container_repositories_count: 100,
container_repositories_synced_count: 50,
container_repositories_failed_count: 12,
design_repositories_count: 100,
design_repositories_synced_count: 50,
design_repositories_failed_count: 12,
attachments_count: 30,
attachments_synced_count: 30,
attachments_failed_count: 25,
......
......@@ -40,6 +40,9 @@ describe Geo::DesignRepositorySyncService do
allow_any_instance_of(Geo::ProjectHousekeepingService).to receive(:execute)
.and_return(nil)
allow_any_instance_of(Users::RefreshAuthorizedProjectsService).to receive(:execute)
.and_return(nil)
end
include_context 'lease handling'
......
......@@ -33,6 +33,9 @@ describe Geo::MetricsUpdateService, :geo, :prometheus do
container_repositories_count: 100,
container_repositories_synced_count: 50,
container_repositories_failed_count: 12,
design_repositories_count: 100,
design_repositories_synced_count: 50,
design_repositories_failed_count: 12,
attachments_count: 30,
attachments_synced_count: 30,
attachments_failed_count: 25,
......
......@@ -7493,6 +7493,9 @@ msgstr ""
msgid "GeoNodes|Data replication lag"
msgstr ""
msgid "GeoNodes|Design repositories"
msgstr ""
msgid "GeoNodes|Does not match the primary storage configuration"
msgstr ""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment