Merge branch '3256-improve-replication-status-2' into 'master'

Geo: Improve replication status. Using pg_stat_wal_receiver

Closes #3256

See merge request gitlab-org/gitlab-ee!4187

changelogs/unreleased-ee/3256-improve-replication-status-2.yml

+---
+title: 'Geo: Improve replication status. Using pg_stat_wal_receiver'
+merge_request:
+author:
+type: other

ee/app/views/admin/geo_nodes/index.html.haml

@@ -18,6 +18,10 @@
   %strong exact order
   they appear.
 
+- unless Gitlab::Database.pg_stat_wal_receiver_supported?
+  = content_for :flash_message do
+    .alert.alert-warning WARNING: Please upgrade PostgreSQL to version 9.6 or greater. The status of the replication cannot be determined reliably with the current version.
+
 - if @nodes.any?
   #js-geo-nodes{ data: { primary_version: "#{Gitlab::VERSION}", primary_revision: "#{Gitlab::REVISION}", node_details_path: "#{admin_geo_nodes_path}", node_actions_allowed: "#{Gitlab::Database.read_write?}", node_edit_allowed: "#{Gitlab::Geo.license_allows?}" } }
 - else

ee/lib/gitlab/geo/health_check.rb

@@ -7,7 +7,7 @@ module Gitlab
         return '' unless Gitlab::Geo.secondary?
         return 'The Geo database configuration file is missing.' unless Gitlab::Geo.geo_database_configured?
         return 'The Geo node has a database that is not configured for streaming replication with the primary node.' unless self.database_secondary?
-        return 'The Geo node does not appear to be replicating data from the primary node.' unless self.db_replication_lag_seconds.present?
+        return 'The Geo node does not appear to be replicating data from the primary node.' if Gitlab::Database.pg_stat_wal_receiver_supported? && !self.streaming_active?
 
         database_version  = self.get_database_version.to_i
         migration_version = self.get_migration_version.to_i
@@ -76,6 +76,18 @@ module Gitlab
 
         lag.present? ? lag.to_i : lag
       end
+
+      def self.streaming_active?
+        # Get a streaming status
+        # This only works for Postgresql 9.6 and greater
+        status =
+          ActiveRecord::Base.connection.execute('
+          SELECT * FROM pg_stat_wal_receiver;')
+          .first
+          .fetch('status')
+
+        status == 'streaming'
+      end
     end
   end
 end

ee/lib/tasks/geo.rake

@@ -219,6 +219,12 @@ namespace :geo do
     puts GeoNode.current_node_url
     puts '-----------------------------------------------------'.color(:yellow)
 
+    unless Gitlab::Database.pg_stat_wal_receiver_supported?
+      puts
+      puts 'WARNING: Please upgrade PostgreSQL to version 9.6 or greater. The status of the replication cannot be determined reliably with the current version.'.color(:red)
+      puts
+    end
+
     print 'GitLab version: '.rjust(COLUMN_WIDTH)
     puts Gitlab::VERSION
 

lib/gitlab/database.rb

@@ -54,6 +54,10 @@ module Gitlab
       postgresql? && version.to_f >= 9.4
     end
 
+    def self.pg_stat_wal_receiver_supported?
+      postgresql? && version.to_f >= 9.6
+    end
+
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 

spec/ee/spec/lib/gitlab/geo/health_check_spec.rb

@@ -15,7 +15,7 @@ describe Gitlab::Geo::HealthCheck, :postgresql do
       allow(described_class).to receive(:database_secondary?).and_return(true)
       allow(described_class).to receive(:get_database_version).and_return('20170101')
       allow(described_class).to receive(:get_migration_version).and_return('20170201')
-      allow(described_class).to receive(:db_replication_lag_seconds).and_return(0)
+      allow(described_class).to receive(:streaming_active?).and_return(true)
 
       message = subject.perform_checks
 
@@ -54,7 +54,7 @@ describe Gitlab::Geo::HealthCheck, :postgresql do
     it 'returns an error when Geo database version does not match the latest migration version' do
       allow(described_class).to receive(:database_secondary?).and_return(true)
       allow(subject).to receive(:get_database_version) { 1 }
-      allow(described_class).to receive(:db_replication_lag_seconds).and_return(0)
+      allow(described_class).to receive(:streaming_active?).and_return(true)
 
       expect(subject.perform_checks).to match(/Current Geo database version \([0-9]+\) does not match latest migration \([0-9]+\)/)
     end
@@ -62,17 +62,26 @@ describe Gitlab::Geo::HealthCheck, :postgresql do
     it 'returns an error when latest migration version does not match the Geo database version' do
       allow(described_class).to receive(:database_secondary?).and_return(true)
       allow(subject).to receive(:get_migration_version) { 1 }
-      allow(described_class).to receive(:db_replication_lag_seconds).and_return(0)
+      allow(described_class).to receive(:streaming_active?).and_return(true)
 
       expect(subject.perform_checks).to match(/Current Geo database version \([0-9]+\) does not match latest migration \([0-9]+\)/)
     end
 
-    it 'returns an error when replication lag is not present' do
+    it 'returns an error when streaming is not active and Postgresql supports pg_stat_wal_receiver' do
+      allow(Gitlab::Database).to receive(:pg_stat_wal_receiver_supported?).and_return(true)
       allow(described_class).to receive(:database_secondary?).and_return(true)
-      allow(described_class).to receive(:db_replication_lag_seconds).and_return(nil)
+      allow(described_class).to receive(:streaming_active?).and_return(false)
 
       expect(subject.perform_checks).to match(/The Geo node does not appear to be replicating data from the primary node/)
     end
+
+    it 'returns an error when streaming is not active and Postgresql does not support pg_stat_wal_receiver' do
+      allow(Gitlab::Database).to receive(:pg_stat_wal_receiver_supported?).and_return(false)
+      allow(described_class).to receive(:database_secondary?).and_return(true)
+      allow(described_class).to receive(:streaming_active?).and_return(false)
+
+      expect(subject.perform_checks).to be_empty
+    end
   end
 
   describe 'MySQL checks' do