Commit e9c5d70a authored by Stan Hu's avatar Stan Hu

Merge branch 'tc-geo-upload-verification' into 'master'

Geo admin panel for upload verification

Closes #5591

See merge request gitlab-org/gitlab-ee!9720
parents 8f06de93 d49bb91a
......@@ -170,27 +170,7 @@
%strong.fly-out-top-item-name
= _('Push Rules')
= nav_link(controller: %w(admin/geo/nodes admin/geo/projects)) do
= link_to admin_geo_nodes_path, class: "qa-link-geo-menu" do
.nav-icon-container
= sprite_icon('location-dot')
%span.nav-item-name
#{ _('Geo') }
- if Gitlab::Geo.secondary?
%ul.sidebar-sub-level-items
= nav_link(controller: 'admin/geo/nodes', html_options: { class: "fly-out-top-item" } ) do
= link_to admin_geo_nodes_path do
%strong.fly-out-top-item-name
#{ _('Geo') }
%li.divider.fly-out-top-item
= nav_link(path: 'admin/geo/nodes#index') do
= link_to admin_geo_nodes_path, title: 'Nodes' do
%span
#{ _('Nodes') }
= nav_link(path: 'admin/geo/projects#index') do
= link_to admin_geo_projects_path, title: 'Projects' do
%span
= _('Projects')
= render 'layouts/nav/ee/admin/geo_sidebar'
= nav_link(controller: :deploy_keys) do
= link_to admin_deploy_keys_path do
......
......@@ -178,13 +178,9 @@
}
.geo-admin-projects,
.geo-admin-uploads,
.admin-projects {
.card-header {
.header-text-primary,
.header-text-secondary {
color: $blue-600;
}
.header-text-primary {
line-height: 28px;
text-overflow: ellipsis;
......@@ -213,12 +209,13 @@
}
.card-body {
.project-container {
.project-container,
.upload-container {
margin-left: 0;
padding-left: 0;
}
.project-status-content {
.geo-status-content {
&.status-type-success {
color: $green-600;
}
......@@ -229,7 +226,8 @@
}
@include media-breakpoint-down(xs) {
.project-status-container + .project-status-container {
.project-status-container + .project-status-container,
.upload-status-container + .upload-status-container {
margin-top: 15px;
}
}
......
# frozen_string_literal: true
class Admin::Geo::ApplicationController < Admin::ApplicationController
helper ::EE::GeoHelper
protected
def check_license!
unless Gitlab::Geo.license_allows?
flash[:alert] = _('You need a different license to use Geo replication.')
redirect_to admin_license_path
end
end
end
# frozen_string_literal: true
class Admin::Geo::NodesController < Admin::ApplicationController
before_action :check_license, except: :index
class Admin::Geo::NodesController < Admin::Geo::ApplicationController
before_action :check_license!, except: :index
before_action :load_node, only: [:edit, :update]
helper EE::GeoHelper
# rubocop: disable CodeReuse/ActiveRecord
def index
@nodes = GeoNode.all.order(:id)
@node = GeoNode.new
unless Gitlab::Geo.license_allows?
flash.now[:alert] = _('You need a different license to enable Geo replication.')
flash.now[:alert] = _('You need a different license to use Geo replication.')
end
unless Gitlab::Database.postgresql_minimum_supported_version?
......@@ -62,13 +60,6 @@ class Admin::Geo::NodesController < Admin::ApplicationController
)
end
def check_license
unless Gitlab::Geo.license_allows?
flash[:alert] = 'You need a different license to enable Geo replication'
redirect_to admin_license_path
end
end
def load_node
@node = GeoNode.find(params[:id])
end
......
# frozen_string_literal: true
class Admin::Geo::ProjectsController < Admin::ApplicationController
before_action :check_license
class Admin::Geo::ProjectsController < Admin::Geo::ApplicationController
before_action :check_license!
before_action :load_registry, except: [:index]
before_action :limited_actions_message!
helper ::EE::GeoHelper
def index
finder = ::Geo::ProjectRegistryStatusFinder.new
@registries = case params[:sync_status]
when 'never'
finder.never_synced_projects.page(params[:page])
......@@ -70,12 +66,6 @@ class Admin::Geo::ProjectsController < Admin::ApplicationController
private
def check_license
unless Gitlab::Geo.license_allows?
redirect_to admin_license_path, alert: s_('Geo|You need a different license to use Geo replication')
end
end
def load_registry
@registry = ::Geo::ProjectRegistry.find_by_id(params[:id])
end
......@@ -83,4 +73,8 @@ class Admin::Geo::ProjectsController < Admin::ApplicationController
def redirect_back_or_admin_geo_projects(params)
redirect_back_or_default(default: admin_geo_projects_path, options: params)
end
def finder
@finder ||= ::Geo::ProjectRegistryStatusFinder.new
end
end
# frozen_string_literal: true
class Admin::Geo::UploadsController < Admin::Geo::ApplicationController
before_action :check_license!
before_action :registries, only: [:index]
def index
end
def destroy
if registry.upload
return redirect_admin_geo_uploads(alert: s_('Geo|Could not remove tracking entry for an existing upload.'))
end
registry.destroy
redirect_admin_geo_uploads(notice: s_('Geo|Tracking entry for upload (%{type}/%{id}) was successfully removed.') % { type: registry.file_type, id: registry.file_id })
end
private
def registries
@registries ||=
::Geo::UploadRegistry
.with_status(params[:sync_status])
.with_search(params[:name])
.fresh
.page(params[:page])
end
def registry
@registry ||= ::Geo::UploadRegistry.find_by_id(params[:id])
end
def redirect_admin_geo_uploads(options)
redirect_back_or_default(default: admin_geo_uploads_path, options: options)
end
end
......@@ -92,36 +92,36 @@ module EE
data: data
end
def project_registry_status(project_registry)
status_type = case project_registry.synchronization_state
def geo_registry_status(registry)
status_type = case registry.synchronization_state
when :failed then
'status-type-failure'
when :synced then
'status-type-success'
end
content_tag(:div, class: "project-status-content #{status_type}") do
icon = project_registry_status_icon(project_registry)
text = project_registry_status_text(project_registry)
content_tag(:div, class: "geo-status-content #{status_type}") do
icon = geo_registry_status_icon(registry)
text = geo_registry_status_text(registry)
[icon, text].join(' ').html_safe
end
end
def project_registry_status_icon(project_registry)
icon(STATUS_ICON_NAMES_BY_STATE.fetch(project_registry.synchronization_state, 'exclamation-triangle'))
def geo_registry_status_icon(registry)
icon STATUS_ICON_NAMES_BY_STATE.fetch(registry.synchronization_state, 'exclamation-triangle')
end
def project_registry_status_text(project_registry)
case project_registry.synchronization_state
def geo_registry_status_text(registry)
case registry.synchronization_state
when :never
s_('Geo|Not synced yet')
when :failed
s_('Geo|Failed')
when :pending
if project_registry.pending_synchronization?
if registry.pending_synchronization?
s_('Geo|Pending synchronization')
elsif project_registry.pending_verification?
elsif registry.pending_verification?
s_('Geo|Pending verification')
else
# should never reach this state, unless we introduce new behavior
......
......@@ -3,6 +3,7 @@
module Geo
module Fdw
class Upload < ::Geo::BaseFdw
include Gitlab::SQL::Pattern
include ObjectStorable
STORE_COLUMN = :store
......@@ -10,6 +11,18 @@ module Geo
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('uploads')
scope :geo_syncable, -> { with_files_stored_locally }
class << self
# Searches for a list of uploads based on the query given in `query`.
#
# On PostgreSQL this method uses "ILIKE" to perform a case-insensitive
# search.
#
# query - The search query as a String.
def search(query)
fuzzy_search(query, [:path])
end
end
end
end
end
......@@ -5,6 +5,9 @@ class Geo::FileRegistry < Geo::BaseRegistry
scope :lfs_objects, -> { where(file_type: :lfs) }
scope :attachments, -> { where(file_type: Geo::FileService::DEFAULT_OBJECT_TYPES) }
scope :failed, -> { where(success: false).where.not(retry_count: nil) }
scope :never, -> { where(success: false, retry_count: nil) }
scope :fresh, -> { order(created_at: :desc) }
self.inheritance_column = 'file_type'
......@@ -15,4 +18,26 @@ class Geo::FileRegistry < Geo::BaseRegistry
Geo::UploadRegistry
end
end
def self.with_status(status)
case status
when 'synced', 'never', 'failed'
self.public_send(status) # rubocop: disable GitlabSecurity/PublicSend
else
all
end
end
# Returns a synchronization state based on existing attribute values
#
# It takes into account things like if a successful replication has been done
# if there are pending actions or existing errors
#
# @return [Symbol] :synced, :never, or :failed
def synchronization_state
return :synced if success?
return :never if retry_count.nil?
:failed
end
end
......@@ -14,4 +14,18 @@ class Geo::UploadRegistry < Geo::FileRegistry
sti_column.in(sti_names)
end
def self.with_search(query)
return all if query.nil?
where(file_id: Geo::Fdw::Upload.search(query))
end
def file
upload&.path || s_('Removed %{type} with id %{id}') % { type: file_type, id: file_id }
end
def project
return upload.model if upload&.model.is_a?(Project)
end
end
......@@ -4,7 +4,7 @@
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Status')
= project_registry_status(project_registry)
= geo_registry_status(project_registry)
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Next sync scheduled at')
......
......@@ -4,7 +4,7 @@
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Status')
= project_registry_status(project_registry)
= geo_registry_status(project_registry)
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Next sync scheduled at')
......
......@@ -4,7 +4,7 @@
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Status')
= project_registry_status(project_registry)
= geo_registry_status(project_registry)
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Next sync scheduled at')
......
......@@ -4,7 +4,7 @@
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Status')
= project_registry_status(project_registry)
= geo_registry_status(project_registry)
.col-sm.project-status-container
.project-status-title.text-muted
= s_('Geo|Last successful sync')
......
.card.upload-card.prepend-top-15
.card-header{ id: "upload-#{upload_registry.id}-header" }
.d-flex
%strong.header-text-primary.flex-fill
= upload_registry.file
- unless upload_registry.success?
= link_to('sync_admin_geo_upload_path(upload_registry)', method: :post, class: 'btn btn-default btn-sm mr-2') do
= s_('Geo|Sync')
- unless upload_registry.upload
= link_to(admin_geo_upload_path(upload_registry), data: { confirm: s_('Geo|Tracking entry will be removed. Are you sure?')}, method: :delete, class: 'btn btn-inverted btn-remove btn-sm') do
= s_('Geo|Remove')
.card-body
.container.upload-container
.row
.col-sm.upload-status-container
.upload-status-title.text-muted
= s_('Geo|Status')
= geo_registry_status(upload_registry)
.col-sm.upload-status-container
.upload-status-title.text-muted
= s_('Geo|Synced at')
.upload-status-content
- if upload_registry.success?
= time_ago_with_tooltip(upload_registry.created_at, placement: 'bottom')
- else
= s_('Geo|Never')
- if upload_registry.project
.col-sm.upload-status-container
.upload-status-title.text-muted
= s_('Geo|Project')
.upload-status-content
= link_to(upload_registry.project.full_name, admin_namespace_project_path(upload_registry.project.namespace, upload_registry.project))
- page_title 'Geo Uploads'
- @content_class = "geo-admin-container geo-admin-uploads"
- params[:sync_status] ||= []
%div{ class: container_class }
.top-area.scrolling-tabs-container.inner-page-scroll-tabs
%ul.nav-links.nav.nav-tabs
- opts = params[:sync_status].present? ? {} : { page: admin_geo_uploads_path }
= nav_link(opts) do
= link_to admin_geo_uploads_path do
= s_('Geo|All')
= nav_link(html_options: { class: active_when(params[:sync_status] == 'synced') }) do
= link_to admin_geo_uploads_path(sync_status: 'synced') do
= s_('Geo|Synced')
= nav_link(html_options: { class: active_when(params[:sync_status] == 'failed') }) do
= link_to admin_geo_uploads_path(sync_status: 'failed') do
= s_('Geo|Failed')
= nav_link(html_options: { class: active_when(params[:sync_status] == 'never') }) do
= link_to admin_geo_uploads_path(sync_status: 'never') do
= s_('Geo|Never')
.nav-controls
= render(partial: 'shared/projects/search_form', autofocus: true)
- @registries.each do |upload_registry|
= render partial: 'registry', locals: { upload_registry: upload_registry }
= paginate @registries, theme: 'gitlab'
= nav_link(controller: %w(admin/geo/nodes admin/geo/projects admin/geo/uploads)) do
= link_to admin_geo_nodes_path, class: "qa-link-geo-menu" do
.nav-icon-container
= sprite_icon('location-dot')
%span.nav-item-name
#{ _('Geo') }
- if Gitlab::Geo.secondary?
%ul.sidebar-sub-level-items
= nav_link(controller: 'admin/geo/nodes', html_options: { class: "fly-out-top-item" } ) do
= link_to admin_geo_nodes_path do
%strong.fly-out-top-item-name
#{ _('Geo') }
%li.divider.fly-out-top-item
= nav_link(path: 'admin/geo/nodes#index') do
= link_to admin_geo_nodes_path, title: 'Nodes' do
%span
#{ _('Nodes') }
= nav_link(path: 'admin/geo/projects#index') do
= link_to admin_geo_projects_path, title: 'Projects' do
%span
= _('Projects')
= nav_link(path: 'admin/geo/uploads#index') do
= link_to admin_geo_uploads_path, title: 'Uploads' do
%span
= _('Uploads')
---
title: Geo admin panel for upload verification
merge_request: 9720
author:
type: added
......@@ -40,6 +40,8 @@ namespace :admin do
post :resync_all
end
end
resources :uploads, only: [:index, :destroy]
end
get '/dashboard/stats', to: 'dashboard#stats'
......
......@@ -8,7 +8,7 @@ describe Admin::Geo::NodesController, :postgresql do
end
it 'displays a flash message' do
expect(controller).to set_flash[:alert].to('You need a different license to enable Geo replication')
expect(controller).to set_flash[:alert].to('You need a different license to use Geo replication.')
end
end
......@@ -55,7 +55,7 @@ describe Admin::Geo::NodesController, :postgresql do
allow(Gitlab::Geo).to receive(:license_allows?).and_return(false)
end
it_behaves_like 'with flash message', :alert, 'You need a different license to enable Geo replication'
it_behaves_like 'with flash message', :alert, 'You need a different license to use Geo replication'
it 'does not redirects to the license page' do
go
......
# frozen_string_literal: true
require 'spec_helper'
describe Admin::Geo::UploadsController, :geo do
include EE::GeoHelpers
set(:admin) { create(:admin) }
set(:secondary) { create(:geo_node) }
set(:synced_registry) { create(:geo_file_registry, :with_file, success: true) }
set(:failed_registry) { create(:geo_file_registry, :failed) }
set(:never_registry) { create(:geo_file_registry, :failed, retry_count: nil) }
def css_id(registry)
"#upload-#{registry.id}-header"
end
before do
sign_in(admin)
end
shared_examples 'license required' do
context 'without a valid license' do
it 'redirects to license page with a flash message' do
expect(subject).to redirect_to(admin_license_path)
expect(flash[:alert]).to include('You need a different license to use Geo replication')
end
end
end
describe '#index' do
subject { get :index }
it_behaves_like 'license required'
context 'with a valid license' do
render_views
before do
stub_licensed_features(geo: true)
stub_current_geo_node(secondary)
end
it 'renders the index template' do
expect(subject).to have_gitlab_http_status(200)
expect(subject).to render_template(:index)
end
context 'without sync_status specified' do
it 'renders all registries' do
expect(subject).to have_gitlab_http_status(200)
expect(response.body).to have_css(css_id(synced_registry))
expect(response.body).to have_css(css_id(failed_registry))
expect(response.body).to have_css(css_id(never_registry))
end
end
context 'with sync_status=synced' do
subject { get :index, params: { sync_status: 'synced' } }
it 'renders only synced registries' do
expect(subject).to have_gitlab_http_status(200)
expect(response.body).to have_css(css_id(synced_registry))
expect(response.body).not_to have_css(css_id(failed_registry))
expect(response.body).not_to have_css(css_id(never_registry))
end
end
context 'with sync_status=failed' do
subject { get :index, params: { sync_status: 'failed' } }
it 'renders only failed registries' do
expect(subject).to have_gitlab_http_status(200)
expect(response.body).not_to have_css(css_id(synced_registry))
expect(response.body).to have_css(css_id(failed_registry))
expect(response.body).not_to have_css(css_id(never_registry))
end
end
context 'with sync_status=never' do
subject { get :index, params: { sync_status: 'never' } }
it 'renders only never synced registries' do
expect(subject).to have_gitlab_http_status(200)
expect(response.body).not_to have_css(css_id(synced_registry))
expect(response.body).not_to have_css(css_id(failed_registry))
expect(response.body).to have_css(css_id(never_registry))
end
end
end
end
describe '#destroy' do
subject { delete :destroy, params: { id: upload_registry } }
it_behaves_like 'license required' do
let(:upload_registry) { create(:geo_file_registry) }
end
context 'with a valid license' do
before do
stub_licensed_features(geo: true)
end
context 'with an orphaned registry' do
let(:upload_registry) { create(:geo_file_registry, success: true) }
it 'removes the registry' do
upload_registry.update_column(:file_id, -1)
expect(subject).to redirect_to(admin_geo_uploads_path)
expect(flash[:notice]).to include('was successfully removed')
expect { Geo::UploadRegistry.find(upload_registry.id) }.to raise_error(ActiveRecord::RecordNotFound)
end
end
context 'with a regular registry' do
let(:upload_registry) { create(:geo_file_registry, :avatar, :with_file, success: true) }
it 'does not delete the registry and gives an error' do
expect(subject).to redirect_to(admin_geo_uploads_path)
expect(flash[:alert]).to include('Could not remove tracking entry')
expect { Geo::UploadRegistry.find(upload_registry.id) }.not_to raise_error
end
end
end
end
end
......@@ -13,6 +13,13 @@ FactoryBot.define do
trait(:favicon) { file_type :favicon }
trait(:import_export) { file_type :import_export }
factory :geo_upload_registry, class: Geo::UploadRegistry
trait :failed do
success false
retry_count 1
end
trait :with_file do
after(:build, :stub) do |registry, _|
file =
......
......@@ -87,7 +87,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that have been synced' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_2.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id)
......@@ -114,7 +114,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that has been synced' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_2.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id)
......@@ -142,7 +142,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that has been synced' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_2.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id)
......@@ -177,17 +177,17 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that sync has failed' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_2.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_3.id)
expect(subject.count_failed).to eq 2
end
it 'ignores remote attachments' do
create(:geo_file_registry, :attachment, file_id: upload_remote_1.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_2.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_3.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_remote_1.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_2.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_3.id)
expect(subject.count_failed).to eq 2
end
......@@ -204,22 +204,22 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that sync has failed' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id)
expect(subject.count_failed).to eq 1
end
it 'does not count attachments of unsynced projects' do
create(:geo_file_registry, :attachment, file_id: upload_2.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_2.id)
expect(subject.count_failed).to eq 0
end
it 'ignores remote attachments' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_2.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_3.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_2.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_3.id)
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1
......@@ -238,22 +238,22 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'counts attachments that sync has failed' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, file_id: upload_3.id)
expect(subject.count_failed).to eq 1
end
it 'does not count attachments of unsynced projects' do
create(:geo_file_registry, :attachment, file_id: upload_2.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_2.id)
expect(subject.count_failed).to eq 0
end
it 'ignores remote attachments' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_2.id, success: false)
create(:geo_file_registry, :attachment, file_id: upload_3.id, success: false)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_2.id)
create(:geo_file_registry, :attachment, :failed, file_id: upload_3.id)
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1
......@@ -292,7 +292,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'excludes attachments that are not synced' do
create(:geo_file_registry, :attachment, file_id: upload_1.id, success: false, missing_on_primary: true)
create(:geo_file_registry, :attachment, :failed, file_id: upload_1.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 0
end
......@@ -340,7 +340,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it 'returns uploads without an entry on the tracking database' do
create(:geo_file_registry, :avatar, file_id: upload_1.id, success: true)
create(:geo_file_registry, :avatar, file_id: upload_1.id)
uploads = subject.find_unsynced(batch_size: 10)
......@@ -397,8 +397,8 @@ describe Geo::AttachmentRegistryFinder, :geo do
it 'excludes except_file_ids' do
upload_a = create(:upload, :object_storage, model: synced_group)
upload_b = create(:upload, :object_storage, model: unsynced_group)
create(:geo_file_registry, :avatar, file_id: upload_a.id, success: true)
create(:geo_file_registry, :avatar, file_id: upload_b.id, success: true)
create(:geo_file_registry, :avatar, file_id: upload_a.id)
create(:geo_file_registry, :avatar, file_id: upload_b.id)
uploads = subject.find_migrated_local(batch_size: 10, except_file_ids: [upload_a.id])
......
......@@ -87,7 +87,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'counts LFS objects that has been synced' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
......@@ -120,7 +120,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'counts LFS objects that has been synced' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
......@@ -156,17 +156,17 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'counts LFS objects that sync has failed' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
expect(subject.count_failed).to eq 2
end
it 'ignores remote LFS objects' do
create(:geo_file_registry, :lfs, file_id: lfs_object_remote_1.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_remote_1.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
expect(subject.count_failed).to eq 2
end
......@@ -189,17 +189,17 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'counts LFS objects that sync has failed' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
expect(subject.count_failed).to eq 1
end
it 'ignores remote LFS objects' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_2.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_2.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
lfs_object_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1
......@@ -238,7 +238,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'excludes LFS objects that are not synced' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false, missing_on_primary: true)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_1.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 0
end
......@@ -292,8 +292,8 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'returns LFS objects without an entry on the tracking database' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: true)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
lfs_objects = subject.find_unsynced(batch_size: 10)
......@@ -301,8 +301,8 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it 'excludes LFS objects without an entry on the tracking database' do
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: true)
create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
create(:geo_file_registry, :lfs, file_id: lfs_object_1.id)
create(:geo_file_registry, :lfs, :failed, file_id: lfs_object_3.id)
lfs_objects = subject.find_unsynced(batch_size: 10, except_file_ids: [lfs_object_2.id])
......
require 'spec_helper'
describe Geo::FileRegistry do
set(:failed) { create(:geo_file_registry, success: false) }
set(:synced) { create(:geo_file_registry, success: true) }
set(:failed) { create(:geo_file_registry, :failed) }
set(:synced) { create(:geo_file_registry) }
describe '.failed' do
it 'returns registries in the failed state' do
......@@ -17,11 +17,54 @@ describe Geo::FileRegistry do
end
describe '.retry_due' do
set(:retry_yesterday) { create(:geo_file_registry, retry_at: Date.yesterday) }
set(:retry_tomorrow) { create(:geo_file_registry, retry_at: Date.tomorrow) }
it 'returns registries in the synced state' do
retry_yesterday = create(:geo_file_registry, retry_at: Date.yesterday)
create(:geo_file_registry, retry_at: Date.tomorrow)
expect(described_class.retry_due).to match_ids([failed, synced, retry_yesterday])
end
end
describe '.never' do
it 'returns registries that are never synced' do
never = create(:geo_file_registry, retry_count: nil, success: false)
expect(described_class.never).to match_ids([never])
end
end
describe '.with_status' do
it 'finds the registries with status "synced"' do
expect(described_class).to receive(:synced)
described_class.with_status('synced')
end
it 'finds the registries with status "never"' do
expect(described_class).to receive(:never)
described_class.with_status('never')
end
it 'finds the registries with status "failed"' do
expect(described_class).to receive(:failed)
described_class.with_status('failed')
end
end
describe '#synchronization_state' do
it 'returns :synced for a successful synced registry' do
expect(synced.synchronization_state).to eq(:synced)
end
it 'returns :never for a successful registry never synced' do
never = build(:geo_file_registry, success: false, retry_count: nil)
expect(never.synchronization_state).to eq(:never)
end
it 'returns :failed for a failed registry' do
expect(failed.synchronization_state).to eq(:failed)
end
end
end
......@@ -2,15 +2,15 @@
require 'spec_helper'
describe Geo::UploadRegistry, :geo do
set(:lfs_registry) { create(:geo_file_registry, :lfs) }
set(:attachment_registry) { create(:geo_file_registry, :attachment, :with_file) }
set(:avatar_registry) { create(:geo_file_registry, :avatar) }
set(:file_registry) { create(:geo_file_registry, :file) }
set(:namespace_file_registry) { create(:geo_file_registry, :namespace_file) }
set(:personal_file_registry) { create(:geo_file_registry, :personal_file) }
set(:favicon_registry) { create(:geo_file_registry, :favicon) }
set(:import_export_registry) { create(:geo_file_registry, :import_export) }
describe Geo::UploadRegistry, :geo, :delete do
let!(:lfs_registry) { create(:geo_file_registry, :lfs) }
let!(:attachment_registry) { create(:geo_file_registry, :attachment, :with_file) }
let!(:avatar_registry) { create(:geo_file_registry, :avatar) }
let!(:file_registry) { create(:geo_file_registry, :file) }
let!(:namespace_file_registry) { create(:geo_file_registry, :namespace_file) }
let!(:personal_file_registry) { create(:geo_file_registry, :personal_file) }
let!(:favicon_registry) { create(:geo_file_registry, :favicon) }
let!(:import_export_registry) { create(:geo_file_registry, :import_export) }
it 'finds all upload registries' do
expected = [attachment_registry,
......@@ -27,4 +27,27 @@ describe Geo::UploadRegistry, :geo do
it 'finds associated Upload record' do
expect(described_class.find(attachment_registry.id).upload).to be_an_instance_of(Upload)
end
describe '.with_search', :geo_fdw do
it 'searches registries on path' do
upload = create(:upload, path: 'uploads/-/system/project/avatar/my-awesome-avatar.png')
upload_registry = create(:geo_upload_registry, file_id: upload.id, file_type: :avatar)
expect(described_class.with_search('awesome-avatar')).to match_ids(upload_registry)
end
end
describe '#file' do
it 'returns the path of the upload of a registry' do
registry = create(:geo_upload_registry, :file, :with_file)
expect(registry.file).to eq('uploads/-/system/project/avatar/avatar.jpg')
end
it 'return "removed" message when the upload no longer exists' do
registry = create(:geo_upload_registry, :avatar)
expect(registry.file).to match(/^Removed avatar with id/)
end
end
end
......@@ -131,7 +131,7 @@ describe GeoNodeStatus, :geo, :delete do
create(:geo_file_registry, :avatar, file_id: uploads[0])
create(:geo_file_registry, :avatar, file_id: uploads[1])
create(:geo_file_registry, :avatar, file_id: uploads[2], success: false)
create(:geo_file_registry, :avatar, :failed, file_id: uploads[2])
expect(subject.attachments_synced_count).to eq(2)
end
......@@ -176,7 +176,7 @@ describe GeoNodeStatus, :geo, :delete do
create(:geo_file_registry, :avatar, file_id: uploads[0], missing_on_primary: true)
create(:geo_file_registry, :avatar, file_id: uploads[1])
create(:geo_file_registry, :avatar, file_id: uploads[2], success: false)
create(:geo_file_registry, :avatar, :failed, file_id: uploads[2])
expect(subject.attachments_synced_missing_on_primary_count).to eq(1)
end
......@@ -185,13 +185,13 @@ describe GeoNodeStatus, :geo, :delete do
describe '#attachments_failed_count', :delete do
it 'counts failed avatars, attachment, personal snippets and files' do
# These two should be ignored
create(:geo_file_registry, :lfs, :with_file, success: false)
create(:geo_file_registry, :lfs, :with_file, :failed)
create(:geo_file_registry, :with_file)
create(:geo_file_registry, :with_file, file_type: :personal_file, success: false)
create(:geo_file_registry, :with_file, file_type: :attachment, success: false)
create(:geo_file_registry, :avatar, :with_file, success: false)
create(:geo_file_registry, :with_file, success: false)
create(:geo_file_registry, :with_file, :failed, file_type: :personal_file)
create(:geo_file_registry, :with_file, :failed, file_type: :attachment)
create(:geo_file_registry, :avatar, :with_file, :failed)
create(:geo_file_registry, :with_file, :failed)
expect(subject.attachments_failed_count).to eq(4)
end
......@@ -248,12 +248,12 @@ describe GeoNodeStatus, :geo, :delete do
describe '#lfs_objects_synced_count', :delete do
it 'counts synced LFS objects' do
# These four should be ignored
create(:geo_file_registry, success: false)
create(:geo_file_registry, :failed)
create(:geo_file_registry, :avatar)
create(:geo_file_registry, file_type: :attachment)
create(:geo_file_registry, :lfs, :with_file, success: false)
create(:geo_file_registry, :lfs, :with_file, :failed)
create(:geo_file_registry, :lfs, :with_file, success: true)
create(:geo_file_registry, :lfs, :with_file)
expect(subject.lfs_objects_synced_count).to eq(1)
end
......@@ -264,12 +264,12 @@ describe GeoNodeStatus, :geo, :delete do
describe '#lfs_objects_synced_missing_on_primary_count', :delete do
it 'counts LFS objects marked as synced due to file missing on the primary' do
# These four should be ignored
create(:geo_file_registry, success: false)
create(:geo_file_registry, :failed)
create(:geo_file_registry, :avatar, missing_on_primary: true)
create(:geo_file_registry, file_type: :attachment, missing_on_primary: true)
create(:geo_file_registry, :lfs, :with_file, success: false)
create(:geo_file_registry, :lfs, :with_file, :failed)
create(:geo_file_registry, :lfs, :with_file, success: true, missing_on_primary: true)
create(:geo_file_registry, :lfs, :with_file, missing_on_primary: true)
expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1)
end
......@@ -280,12 +280,12 @@ describe GeoNodeStatus, :geo, :delete do
describe '#lfs_objects_failed_count', :delete do
it 'counts failed LFS objects' do
# These four should be ignored
create(:geo_file_registry, success: false)
create(:geo_file_registry, :avatar, success: false)
create(:geo_file_registry, file_type: :attachment, success: false)
create(:geo_file_registry, :failed)
create(:geo_file_registry, :avatar, :failed)
create(:geo_file_registry, :failed, file_type: :attachment)
create(:geo_file_registry, :lfs, :with_file)
create(:geo_file_registry, :lfs, :with_file, success: false)
create(:geo_file_registry, :lfs, :with_file, :failed)
expect(subject.lfs_objects_failed_count).to eq(1)
end
......@@ -308,14 +308,14 @@ describe GeoNodeStatus, :geo, :delete do
end
it 'returns the right percentage with no group restrictions' do
create(:geo_file_registry, :lfs, file_id: lfs_object_project.lfs_object_id, success: true)
create(:geo_file_registry, :lfs, file_id: lfs_object_project.lfs_object_id)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25)
end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:geo_file_registry, :lfs, file_id: lfs_object_project.lfs_object_id, success: true)
create(:geo_file_registry, :lfs, file_id: lfs_object_project.lfs_object_id)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(50)
end
......@@ -326,7 +326,7 @@ describe GeoNodeStatus, :geo, :delete do
describe '#job_artifacts_synced_count', :delete do
it 'counts synced job artifacts' do
# These should be ignored
create(:geo_file_registry, success: true)
create(:geo_file_registry)
create(:geo_job_artifact_registry, :with_artifact, success: false)
create(:geo_job_artifact_registry, :with_artifact, success: true)
......@@ -340,7 +340,7 @@ describe GeoNodeStatus, :geo, :delete do
describe '#job_artifacts_synced_missing_on_primary_count', :delete do
it 'counts job artifacts marked as synced due to file missing on the primary' do
# These should be ignored
create(:geo_file_registry, success: true, missing_on_primary: true)
create(:geo_file_registry, missing_on_primary: true)
create(:geo_job_artifact_registry, :with_artifact, success: true)
create(:geo_job_artifact_registry, :with_artifact, success: true, missing_on_primary: true)
......@@ -354,9 +354,9 @@ describe GeoNodeStatus, :geo, :delete do
describe '#job_artifacts_failed_count', :delete do
it 'counts failed job artifacts' do
# These should be ignored
create(:geo_file_registry, success: false)
create(:geo_file_registry, :avatar, success: false)
create(:geo_file_registry, file_type: :attachment, success: false)
create(:geo_file_registry, :failed)
create(:geo_file_registry, :avatar, :failed)
create(:geo_file_registry, :attachment, :failed)
create(:geo_job_artifact_registry, :with_artifact, success: true)
create(:geo_job_artifact_registry, :with_artifact, success: false)
......
......@@ -18,4 +18,8 @@ RSpec.configure do |config|
config.around(:each, :geo_tracking_db) do |example|
example.run if Gitlab::Geo.geo_database_configured?
end
config.around(:each, :geo_fdw) do |example|
example.run if Gitlab::Geo::Fdw.enabled?
end
end
......@@ -56,7 +56,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
it 'performs Geo::FileDownloadWorker for failed-sync attachments' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 0, success: false)
create(:geo_file_registry, :avatar, :failed, file_id: upload.id, bytes: 0)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', upload.id).once.and_return(spy)
......@@ -65,7 +65,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
it 'does not perform Geo::FileDownloadWorker for synced attachments' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 1234, success: true)
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 1234)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
......@@ -73,7 +73,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
it 'does not perform Geo::FileDownloadWorker for synced attachments even with 0 bytes downloaded' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 0, success: true)
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 0)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
......@@ -81,7 +81,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
context 'with a failed file' do
let(:failed_registry) { create(:geo_file_registry, :avatar, file_id: 999, success: false) }
let(:failed_registry) { create(:geo_file_registry, :avatar, :failed, file_id: 999) }
it 'does not stall backfill' do
unsynced = create(:upload)
......@@ -101,7 +101,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
it 'does not retry failed files when retry_at is tomorrow' do
failed_registry = create(:geo_file_registry, :avatar, file_id: 999, success: false, retry_at: Date.tomorrow)
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.tomorrow)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('avatar', failed_registry.file_id)
......@@ -109,7 +109,7 @@ describe Geo::FileDownloadDispatchWorker, :geo do
end
it 'retries failed files when retry_at is in the past' do
failed_registry = create(:geo_file_registry, :avatar, file_id: 999, success: false, retry_at: Date.yesterday)
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.yesterday)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', failed_registry.file_id)
......
......@@ -5043,6 +5043,9 @@ msgstr ""
msgid "Geo|Could not remove tracking entry for an existing project."
msgstr ""
msgid "Geo|Could not remove tracking entry for an existing upload."
msgstr ""
msgid "Geo|Failed"
msgstr ""
......@@ -5094,6 +5097,9 @@ msgstr ""
msgid "Geo|Please refer to Geo Troubleshooting."
msgstr ""
msgid "Geo|Project"
msgstr ""
msgid "Geo|Project (ID: %{project_id}) no longer exists on the primary. It is safe to remove this entry, as this will not remove any data on disk."
msgstr ""
......@@ -5142,9 +5148,15 @@ msgstr ""
msgid "Geo|Status"
msgstr ""
msgid "Geo|Sync"
msgstr ""
msgid "Geo|Synced"
msgstr ""
msgid "Geo|Synced at"
msgstr ""
msgid "Geo|Synchronization failed - %{error}"
msgstr ""
......@@ -5157,6 +5169,9 @@ msgstr ""
msgid "Geo|Tracking entry for project (%{project_id}) was successfully removed."
msgstr ""
msgid "Geo|Tracking entry for upload (%{type}/%{id}) was successfully removed."
msgstr ""
msgid "Geo|Tracking entry will be removed. Are you sure?"
msgstr ""
......@@ -5181,9 +5196,6 @@ msgstr ""
msgid "Geo|You are on a secondary, <b>read-only</b> Geo node. You may be able to make a limited amount of changes or perform a limited amount of actions on this page."
msgstr ""
msgid "Geo|You need a different license to use Geo replication"
msgstr ""
msgid "Geo|misconfigured"
msgstr ""
......@@ -8936,6 +8948,9 @@ msgstr ""
msgid "Removed"
msgstr ""
msgid "Removed %{type} with id %{id}"
msgstr ""
msgid "Removed group can not be restored!"
msgstr ""
......@@ -11734,6 +11749,9 @@ msgstr ""
msgid "Uploaded on"
msgstr ""
msgid "Uploads"
msgstr ""
msgid "Upstream"
msgstr ""
......@@ -12483,7 +12501,7 @@ msgstr ""
msgid "You need a different license to enable FileLocks feature"
msgstr ""
msgid "You need a different license to enable Geo replication."
msgid "You need a different license to use Geo replication."
msgstr ""
msgid "You need git-lfs version %{min_git_lfs_version} (or greater) to continue. Please visit https://git-lfs.github.com"
......
......@@ -9,9 +9,12 @@ module QA
page.module_eval do
view 'app/views/layouts/nav/sidebar/_admin.html.haml' do
element :link_license_menu
element :link_geo_menu
element :admin_settings_template_item
end
view 'ee/app/views/layouts/nav/ee/admin/_geo_sidebar.html.haml' do
element :link_geo_menu
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment