Commit 818f0655 authored by Robert Speicher's avatar Robert Speicher

Merge branch '2181-fix-secondary-node-settings-changed' into 'master'

Geo - Properly set tracking database connection and cron jobs on secondary node

Closes #2181 and #2289

See merge request !1987
parents cbdf4639 901deb24
class Geo::BaseRegistry < ActiveRecord::Base class Geo::BaseRegistry < ActiveRecord::Base
self.abstract_class = true self.abstract_class = true
if Gitlab::Geo.configured? && (Gitlab::Geo.secondary? || Rails.env.test?) if Gitlab::Geo.secondary_role_enabled?
establish_connection Rails.configuration.geo_database establish_connection Rails.configuration.geo_database
end end
end end
...@@ -21,9 +21,7 @@ class GeoNode < ActiveRecord::Base ...@@ -21,9 +21,7 @@ class GeoNode < ActiveRecord::Base
validates :encrypted_secret_access_key, presence: true validates :encrypted_secret_access_key, presence: true
after_initialize :build_dependents after_initialize :build_dependents
after_save :refresh_bulk_notify_worker_status
after_save :expire_cache! after_save :expire_cache!
after_destroy :refresh_bulk_notify_worker_status
after_destroy :expire_cache! after_destroy :expire_cache!
before_validation :update_dependents_attributes before_validation :update_dependents_attributes
...@@ -130,10 +128,6 @@ class GeoNode < ActiveRecord::Base ...@@ -130,10 +128,6 @@ class GeoNode < ActiveRecord::Base
{ protocol: schema, host: host, port: port, script_name: relative_url } { protocol: schema, host: host, port: port, script_name: relative_url }
end end
def refresh_bulk_notify_worker_status
Gitlab::Geo.configure_cron_jobs!
end
def build_dependents def build_dependents
unless persisted? unless persisted?
self.build_geo_node_key if geo_node_key.nil? self.build_geo_node_key if geo_node_key.nil?
......
...@@ -22,7 +22,7 @@ class GeoFileDownloadDispatchWorker ...@@ -22,7 +22,7 @@ class GeoFileDownloadDispatchWorker
# files, excluding ones in progress. # files, excluding ones in progress.
# 5. Quit when we have scheduled all downloads or exceeded an hour. # 5. Quit when we have scheduled all downloads or exceeded an hour.
def perform def perform
return unless Gitlab::Geo.configured? return unless Gitlab::Geo.secondary_role_enabled?
return unless Gitlab::Geo.secondary? return unless Gitlab::Geo.secondary?
@start_time = Time.now @start_time = Time.now
...@@ -153,7 +153,7 @@ class GeoFileDownloadDispatchWorker ...@@ -153,7 +153,7 @@ class GeoFileDownloadDispatchWorker
def node_enabled? def node_enabled?
# Only check every minute to avoid polling the DB excessively # Only check every minute to avoid polling the DB excessively
unless @last_enabled_check.present? && (Time.now - @last_enabled_check > 1.minute) unless @last_enabled_check.present? && @last_enabled_check > 1.minute.ago
@last_enabled_check = Time.now @last_enabled_check = Time.now
@current_node_enabled = nil @current_node_enabled = nil
end end
......
...@@ -7,7 +7,7 @@ class GeoRepositorySyncWorker ...@@ -7,7 +7,7 @@ class GeoRepositorySyncWorker
LAST_SYNC_INTERVAL = 24.hours LAST_SYNC_INTERVAL = 24.hours
def perform def perform
return unless Gitlab::Geo.configured? return unless Gitlab::Geo.secondary_role_enabled?
return unless Gitlab::Geo.primary_node.present? return unless Gitlab::Geo.primary_node.present?
start_time = Time.now start_time = Time.now
...@@ -20,7 +20,7 @@ class GeoRepositorySyncWorker ...@@ -20,7 +20,7 @@ class GeoRepositorySyncWorker
project_ids.each do |project_id| project_ids.each do |project_id|
begin begin
break if over_time?(start_time) break if over_time?(start_time)
break unless Gitlab::Geo.current_node_enabled? break unless node_enabled?
# We try to obtain a lease here for the entire sync process because we # We try to obtain a lease here for the entire sync process because we
# want to sync the repositories continuously at a controlled rate # want to sync the repositories continuously at a controlled rate
...@@ -73,6 +73,16 @@ class GeoRepositorySyncWorker ...@@ -73,6 +73,16 @@ class GeoRepositorySyncWorker
Time.now - start_time >= RUN_TIME Time.now - start_time >= RUN_TIME
end end
def node_enabled?
# Only check every minute to avoid polling the DB excessively
unless @last_enabled_check.present? && @last_enabled_check > 1.minute.ago
@last_enabled_check = Time.now
@current_node_enabled = nil
end
@current_node_enabled ||= Gitlab::Geo.current_node_enabled?
end
def try_obtain_lease def try_obtain_lease
lease = Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout).try_obtain lease = Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout).try_obtain
......
---
title: Geo - Properly set tracking database connection and cron jobs on secondary nodes
merge_request:
author:
...@@ -622,6 +622,12 @@ production: &base ...@@ -622,6 +622,12 @@ production: &base
# host: localhost # host: localhost
# port: 3808 # port: 3808
## GitLab Geo settings (EE-only)
geo_primary_role:
enabled: false
geo_secondary_role:
enabled: false
# #
# 5. Extra customization # 5. Extra customization
# ========================== # ==========================
...@@ -705,6 +711,10 @@ test: ...@@ -705,6 +711,10 @@ test:
user_filter: '' user_filter: ''
group_base: 'ou=groups,dc=example,dc=com' group_base: 'ou=groups,dc=example,dc=com'
admin_group: '' admin_group: ''
geo_primary_role:
enabled: true
geo_secondary_role:
enabled: true
staging: staging:
<<: *base <<: *base
...@@ -342,6 +342,10 @@ Settings.pages['external_https'] ||= false unless Settings.pages['external_http ...@@ -342,6 +342,10 @@ Settings.pages['external_https'] ||= false unless Settings.pages['external_http
# Geo # Geo
# #
Settings.gitlab['geo_status_timeout'] ||= 10 Settings.gitlab['geo_status_timeout'] ||= 10
Settings['geo_primary_role'] ||= Settingslogic.new({})
Settings.geo_primary_role['enabled'] = false if Settings.geo_primary_role['enabled'].nil?
Settings['geo_secondary_role'] ||= Settingslogic.new({})
Settings.geo_secondary_role['enabled'] = false if Settings.geo_secondary_role['enabled'].nil?
# #
# Git LFS # Git LFS
......
if File.exist?(Rails.root.join('config/database_geo.yml')) if Gitlab::Geo.secondary_role_enabled?
Rails.application.configure do Rails.application.configure do
config.geo_database = config_for(:database_geo) config.geo_database = config_for(:database_geo)
end end
......
...@@ -35,7 +35,6 @@ module API ...@@ -35,7 +35,6 @@ module API
get 'status' do get 'status' do
authenticate_by_gitlab_geo_node_token! authenticate_by_gitlab_geo_node_token!
require_node_to_be_secondary! require_node_to_be_secondary!
require_node_to_have_tracking_db!
present GeoNodeStatus.new(id: Gitlab::Geo.current_node.id), with: Entities::GeoNodeStatus present GeoNodeStatus.new(id: Gitlab::Geo.current_node.id), with: Entities::GeoNodeStatus
end end
...@@ -110,10 +109,6 @@ module API ...@@ -110,10 +109,6 @@ module API
def require_node_to_be_secondary! def require_node_to_be_secondary!
forbidden! 'Geo node is not secondary node.' unless Gitlab::Geo.current_node&.secondary? forbidden! 'Geo node is not secondary node.' unless Gitlab::Geo.current_node&.secondary?
end end
def require_node_to_have_tracking_db!
not_found! 'Geo node does not have its tracking database enabled.' unless Gitlab::Geo.configured?
end
end end
end end
end end
...@@ -42,8 +42,12 @@ module Gitlab ...@@ -42,8 +42,12 @@ module Gitlab
Gitlab::Geo.current_node.reload.enabled? Gitlab::Geo.current_node.reload.enabled?
end end
def self.configured? def self.primary_role_enabled?
Rails.configuration.respond_to?(:geo_database) Gitlab.config.geo_primary_role['enabled']
end
def self.secondary_role_enabled?
Gitlab.config.geo_secondary_role['enabled']
end end
def self.license_allows? def self.license_allows?
...@@ -94,9 +98,9 @@ module Gitlab ...@@ -94,9 +98,9 @@ module Gitlab
end end
def self.configure_cron_jobs! def self.configure_cron_jobs!
if self.primary? if self.primary_role_enabled?
self.configure_primary_jobs! self.configure_primary_jobs!
elsif self.secondary? elsif self.secondary_role_enabled?
self.configure_secondary_jobs! self.configure_secondary_jobs!
else else
self.disable_all_jobs! self.disable_all_jobs!
......
...@@ -3,7 +3,8 @@ module Gitlab ...@@ -3,7 +3,8 @@ module Gitlab
class HealthCheck class HealthCheck
def self.perform_checks def self.perform_checks
return '' unless Gitlab::Geo.secondary? return '' unless Gitlab::Geo.secondary?
return 'The Geo database configuration file is missing.' unless Gitlab::Geo.configured? return 'The Geo secondary role is disabled.' unless Gitlab::Geo.secondary_role_enabled?
return 'The Geo database configuration file is missing.' unless self.geo_database_configured?
return 'The Geo node has a database that is not configured for streaming replication with the primary node.' unless self.database_secondary? return 'The Geo node has a database that is not configured for streaming replication with the primary node.' unless self.database_secondary?
database_version = self.get_database_version.to_i database_version = self.get_database_version.to_i
...@@ -57,6 +58,10 @@ module Gitlab ...@@ -57,6 +58,10 @@ module Gitlab
.first .first
.fetch('pg_is_in_recovery') == 't' .fetch('pg_is_in_recovery') == 't'
end end
def self.geo_database_configured?
Rails.configuration.respond_to?(:geo_database)
end
end end
end end
end end
...@@ -16,6 +16,13 @@ describe Gitlab::Geo::HealthCheck do ...@@ -16,6 +16,13 @@ describe Gitlab::Geo::HealthCheck do
expect(subject.perform_checks).to be_blank expect(subject.perform_checks).to be_blank
end end
it 'returns an error when secondary role is disabled' do
allow(Gitlab::Geo).to receive(:secondary?) { true }
allow(Gitlab::Geo).to receive(:secondary_role_enabled?).and_return(false)
expect(subject.perform_checks).not_to be_blank
end
it 'returns an error when database is not configured for streaming replication' do it 'returns an error when database is not configured for streaming replication' do
allow(Gitlab::Geo).to receive(:secondary?) { true } allow(Gitlab::Geo).to receive(:secondary?) { true }
allow(Gitlab::Database).to receive(:postgresql?) { true } allow(Gitlab::Database).to receive(:postgresql?) { true }
......
...@@ -127,7 +127,9 @@ describe Gitlab::Geo, lib: true do ...@@ -127,7 +127,9 @@ describe Gitlab::Geo, lib: true do
end end
it 'activates cron jobs for primary' do it 'activates cron jobs for primary' do
allow(described_class).to receive(:primary?).and_return(true) allow(described_class).to receive(:primary_role_enabled?).and_return(true)
allow(described_class).to receive(:secondary_role_enabled?).and_return(false)
described_class.configure_cron_jobs! described_class.configure_cron_jobs!
expect(described_class.bulk_notify_job).to be_enabled expect(described_class.bulk_notify_job).to be_enabled
...@@ -136,7 +138,9 @@ describe Gitlab::Geo, lib: true do ...@@ -136,7 +138,9 @@ describe Gitlab::Geo, lib: true do
end end
it 'activates cron jobs for secondary' do it 'activates cron jobs for secondary' do
allow(described_class).to receive(:secondary?).and_return(true) allow(described_class).to receive(:primary_role_enabled?).and_return(false)
allow(described_class).to receive(:secondary_role_enabled?).and_return(true)
described_class.configure_cron_jobs! described_class.configure_cron_jobs!
expect(described_class.bulk_notify_job).not_to be_enabled expect(described_class.bulk_notify_job).not_to be_enabled
...@@ -145,6 +149,9 @@ describe Gitlab::Geo, lib: true do ...@@ -145,6 +149,9 @@ describe Gitlab::Geo, lib: true do
end end
it 'deactivates all jobs when Geo is not active' do it 'deactivates all jobs when Geo is not active' do
allow(described_class).to receive(:primary_role_enabled?).and_return(false)
allow(described_class).to receive(:secondary_role_enabled?).and_return(false)
described_class.configure_cron_jobs! described_class.configure_cron_jobs!
expect(described_class.bulk_notify_job).not_to be_enabled expect(described_class.bulk_notify_job).not_to be_enabled
......
...@@ -307,14 +307,6 @@ describe API::Geo, api: true do ...@@ -307,14 +307,6 @@ describe API::Geo, api: true do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(response).to match_response_schema('geo_node_status') expect(response).to match_response_schema('geo_node_status')
end end
it 'responds with a 404 when the tracking database is disabled' do
allow(Gitlab::Geo).to receive(:configured?).and_return(false)
get api('/geo/status'), nil, request.headers
expect(response).to have_http_status(404)
end
end end
context 'when requesting primary node with valid auth header' do context 'when requesting primary node with valid auth header' do
......
...@@ -13,10 +13,10 @@ describe GeoFileDownloadDispatchWorker do ...@@ -13,10 +13,10 @@ describe GeoFileDownloadDispatchWorker do
subject { described_class.new } subject { described_class.new }
describe '#perform' do describe '#perform' do
it 'does not schedule anything when tracking DB is not available' do it 'does not schedule anything when secondary role is disabled' do
create(:lfs_object, :with_file) create(:lfs_object, :with_file)
allow(Rails.configuration).to receive(:respond_to?).with(:geo_database) { false } allow(Gitlab::Geo).to receive(:secondary_role_enabled?) { false }
expect(GeoFileDownloadWorker).not_to receive(:perform_async) expect(GeoFileDownloadWorker).not_to receive(:perform_async)
......
...@@ -52,8 +52,8 @@ describe GeoRepositorySyncWorker do ...@@ -52,8 +52,8 @@ describe GeoRepositorySyncWorker do
subject.perform subject.perform
end end
it 'does not perform Geo::RepositorySyncService when tracking DB is not available' do it 'does not perform Geo::RepositorySyncService when secondary role is disabled' do
allow(Rails.configuration).to receive(:respond_to?).with(:geo_database) { false } allow(Gitlab::Geo).to receive(:secondary_role_enabled?) { false }
expect(Geo::RepositorySyncService).not_to receive(:new) expect(Geo::RepositorySyncService).not_to receive(:new)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment