Merge branch 'bw-fix-postgresql-10-pg-log' into 'master'

Renaming of "xlog" to "wal" for PostgreSQL 10

See merge request gitlab-org/gitlab-ee!4669

ee/app/models/pg_replication_slot.rb

@@ -20,7 +20,8 @@ class PgReplicationSlot
   # http://bdr-project.org/docs/stable/monitoring-peers.html
   def self.slots_retained_bytes
     ActiveRecord::Base.connection.execute(<<-SQL.squish)
-      SELECT slot_name, database, active, pg_xlog_location_diff(pg_current_xlog_insert_location(), restart_lsn)
+      SELECT slot_name, database,
+             active, #{Gitlab::Database.pg_wal_lsn_diff}(#{Gitlab::Database.pg_current_wal_insert_lsn}(), restart_lsn)
         AS retained_bytes
         FROM pg_replication_slots;
     SQL
@@ -30,7 +31,7 @@ class PgReplicationSlot
   # returns the max number WAL space (in bytes) being used across the replication slots
   def self.max_retained_wal
     ActiveRecord::Base.connection.execute(<<-SQL.squish)
-      SELECT COALESCE(MAX(pg_xlog_location_diff(pg_current_xlog_insert_location(), restart_lsn)), 0)
+      SELECT COALESCE(MAX(#{Gitlab::Database.pg_wal_lsn_diff}(#{Gitlab::Database.pg_current_wal_insert_lsn}(), restart_lsn)), 0)
         FROM pg_replication_slots;
     SQL
     .first.fetch('coalesce').to_i

ee/lib/gitlab/database/load_balancing/host.rb

@@ -63,7 +63,7 @@ module Gitlab
         def data_is_recent_enough?
           # It's possible for a replica to not replay WAL data for a while,
           # despite being up to date. This can happen when a primary does not
-          # receive any writes a for a while.
+          # receive any writes for a while.
           #
           # To prevent this from happening we check if the lag size (in bytes)
           # of the replica is small enough for the replica to be useful. We
@@ -92,7 +92,10 @@ module Gitlab
         # This method will return nil if no lag size could be calculated.
         def replication_lag_size
           location = connection.quote(primary_write_location)
-          row = query_and_release("SELECT pg_xlog_location_diff(#{location}, pg_last_xlog_replay_location())::float AS diff")
+          row = query_and_release(<<-SQL.squish)
+            SELECT #{Gitlab::Database.pg_wal_lsn_diff}(#{location}, #{Gitlab::Database.pg_last_wal_replay_lsn}())::float
+              AS diff
+          SQL
 
           row['diff'].to_i if row.any?
         end
@@ -110,11 +113,14 @@ module Gitlab
         def caught_up?(location)
           string = connection.quote(location)
 
-          # In case the host is a primary pg_last_xlog_replay_location() returns
+          # In case the host is a primary pg_last_wal_replay_lsn/pg_last_xlog_replay_location() returns
           # NULL. The recovery check ensures we treat the host as up-to-date in
           # such a case.
-          query = "SELECT NOT pg_is_in_recovery() OR " \
-            "pg_xlog_location_diff(pg_last_xlog_replay_location(), #{string}) >= 0 AS result"
+          query = <<-SQL.squish
+            SELECT NOT pg_is_in_recovery()
+              OR #{Gitlab::Database.pg_wal_lsn_diff}(#{Gitlab::Database.pg_last_wal_replay_lsn}(), #{string}) >= 0
+              AS result
+          SQL
 
           row = query_and_release(query)
 

ee/lib/gitlab/database/load_balancing/load_balancer.rb

@@ -89,7 +89,7 @@ module Gitlab
         def primary_write_location
           read_write do |connection|
             row = connection
-              .select_all('SELECT pg_current_xlog_insert_location()::text AS location')
+              .select_all("SELECT #{Gitlab::Database.pg_current_wal_insert_lsn}()::text AS location")
               .first
 
             if row

ee/lib/gitlab/geo/health_check.rb

@@ -76,14 +76,15 @@ module Gitlab
       def self.db_replication_lag_seconds
         # Obtain the replication lag in seconds
         lag =
-          ActiveRecord::Base.connection.execute('
-          SELECT CASE
-                 WHEN pg_last_xlog_receive_location() = pg_last_xlog_replay_location()
-                  THEN 0
-                 ELSE
-                  EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())::INTEGER
-                 END
-                 AS replication_lag')
+          ActiveRecord::Base.connection.execute(<<-SQL.squish)
+            SELECT CASE
+                   WHEN #{Gitlab::Database.pg_last_wal_receive_lsn}() = #{Gitlab::Database.pg_last_wal_receive_lsn}()
+                    THEN 0
+                   ELSE
+                    EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())::INTEGER
+                   END
+                   AS replication_lag
+          SQL
           .first
           .fetch('replication_lag')
 

ee/spec/models/pg_replication_slot_spec.rb

@@ -7,9 +7,11 @@ describe PgReplicationSlot, :postgresql do
         expect(described_class.max_replication_slots).to be >= 0
       end
 
-      skip = PgReplicationSlot.max_replication_slots <= PgReplicationSlot.count
-      context 'with enough slots available', skip: (skip ? 'max_replication_slots too small' : nil) do
+      skip_examples = PgReplicationSlot.max_replication_slots <= PgReplicationSlot.count
+      context 'with enough slots available' do
         before(:all) do
+          skip('max_replication_slots too small') if skip_examples
+
           @current_slot_count =
             ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM pg_replication_slots;")
             .first.fetch('count').to_i
@@ -21,7 +23,9 @@ describe PgReplicationSlot, :postgresql do
         end
 
         after(:all) do
-          ActiveRecord::Base.connection.execute("SELECT pg_drop_replication_slot('test_slot');")
+          unless skip_examples
+            ActiveRecord::Base.connection.execute("SELECT pg_drop_replication_slot('test_slot');")
+          end
         end
 
         it '#slots_count' do

lib/gitlab/database.rb

@@ -46,6 +46,10 @@ module Gitlab
       database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
     end
 
+    def self.postgresql_9_or_less?
+      postgresql? && version.to_f < 10
+    end
+
     def self.join_lateral_supported?
       postgresql? && version.to_f >= 9.3
     end
@@ -58,6 +62,24 @@ module Gitlab
       postgresql? && version.to_f >= 9.6
     end
 
+    # map some of the function names that changed between PostgreSQL 9 and 10
+    # https://wiki.postgresql.org/wiki/New_in_postgres_10
+    def self.pg_wal_lsn_diff
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
+    end
+
+    def self.pg_current_wal_insert_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
+    end
+
+    def self.pg_last_wal_receive_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
+    end
+
+    def self.pg_last_wal_replay_lsn
+      Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
+    end
+
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 

spec/lib/gitlab/database_spec.rb

@@ -51,6 +51,28 @@ describe Gitlab::Database do
     end
   end
 
+  describe '.postgresql_9_or_less?' do
+    it 'returns false when using MySQL' do
+      allow(described_class).to receive(:postgresql?).and_return(false)
+
+      expect(described_class.postgresql_9_or_less?).to eq(false)
+    end
+
+    it 'returns true when using PostgreSQL 9.6' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('9.6')
+
+      expect(described_class.postgresql_9_or_less?).to eq(true)
+    end
+
+    it 'returns false when using PostgreSQL 10 or newer' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('10')
+
+      expect(described_class.postgresql_9_or_less?).to eq(false)
+    end
+  end
+
   describe '.join_lateral_supported?' do
     it 'returns false when using MySQL' do
       allow(described_class).to receive(:postgresql?).and_return(false)
@@ -95,6 +117,70 @@ describe Gitlab::Database do
     end
   end
 
+  describe '.pg_wal_lsn_diff' do
+    it 'returns old name when using PostgreSQL 9.6' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('9.6')
+
+      expect(described_class.pg_wal_lsn_diff).to eq('pg_xlog_location_diff')
+    end
+
+    it 'returns new name when using PostgreSQL 10 or newer' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('10')
+
+      expect(described_class.pg_wal_lsn_diff).to eq('pg_wal_lsn_diff')
+    end
+  end
+
+  describe '.pg_current_wal_insert_lsn' do
+    it 'returns old name when using PostgreSQL 9.6' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('9.6')
+
+      expect(described_class.pg_current_wal_insert_lsn).to eq('pg_current_xlog_insert_location')
+    end
+
+    it 'returns new name when using PostgreSQL 10 or newer' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('10')
+
+      expect(described_class.pg_current_wal_insert_lsn).to eq('pg_current_wal_insert_lsn')
+    end
+  end
+
+  describe '.pg_last_wal_receive_lsn' do
+    it 'returns old name when using PostgreSQL 9.6' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('9.6')
+
+      expect(described_class.pg_last_wal_receive_lsn).to eq('pg_last_xlog_receive_location')
+    end
+
+    it 'returns new name when using PostgreSQL 10 or newer' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('10')
+
+      expect(described_class.pg_last_wal_receive_lsn).to eq('pg_last_wal_receive_lsn')
+    end
+  end
+
+  describe '.pg_last_wal_replay_lsn' do
+    it 'returns old name when using PostgreSQL 9.6' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('9.6')
+
+      expect(described_class.pg_last_wal_replay_lsn).to eq('pg_last_xlog_replay_location')
+    end
+
+    it 'returns new name when using PostgreSQL 10 or newer' do
+      allow(described_class).to receive(:postgresql?).and_return(true)
+      allow(described_class).to receive(:version).and_return('10')
+
+      expect(described_class.pg_last_wal_replay_lsn).to eq('pg_last_wal_replay_lsn')
+    end
+  end
+
   describe '.nulls_last_order' do
     context 'when using PostgreSQL' do
       before do