Merge branch 'load-balancer-connection-bug' into 'master'

Catch errors in LoadBalancing::Host#online?

See merge request gitlab-org/gitlab-ee!5177

ee/changelogs/unreleased/load-balancer-connection-bug.yml

+---
+title: Catch errors in LoadBalancing::Host#online?
+merge_request:
+author:
+type: fixed

ee/lib/gitlab/database/load_balancing/host.rb

@@ -7,6 +7,20 @@ module Gitlab
 
         delegate :connection, :release_connection, to: :pool
 
+        CONNECTION_ERRORS =
+          if defined?(PG)
+            [
+              ActionView::Template::Error,
+              ActiveRecord::StatementInvalid,
+              PG::Error
+            ].freeze
+          else
+            [
+              ActionView::Template::Error,
+              ActiveRecord::StatementInvalid
+            ].freeze
+          end
+
         # host - The address of the database.
         # load_balancer - The LoadBalancer that manages this Host.
         def initialize(host, load_balancer)
@@ -36,6 +50,9 @@ module Gitlab
           LoadBalancing.log(:info, "Host #{@host} came back online") if @online
 
           @online
+        rescue *CONNECTION_ERRORS
+          offline!
+          false
         end
 
         def refresh_status

ee/spec/lib/gitlab/database/load_balancing/host_spec.rb

@@ -59,6 +59,36 @@ describe Gitlab::Database::LoadBalancing::Host, :postgresql do
         expect(host).to be_online
       end
     end
+
+    context 'when the replica is not online' do
+      it 'returns false when ActionView::Template::Error is raised' do
+        error = StandardError.new
+
+        allow(host)
+          .to receive(:check_replica_status?)
+          .and_raise(ActionView::Template::Error.new('boom', error))
+
+        expect(host).not_to be_online
+      end
+
+      it 'returns false when ActiveRecord::StatementInvalid is raised' do
+        allow(host)
+          .to receive(:check_replica_status?)
+          .and_raise(ActiveRecord::StatementInvalid.new('foo'))
+
+        expect(host).not_to be_online
+      end
+
+      if Gitlab::Database.postgresql?
+        it 'returns false when PG::Error is raised' do
+          allow(host)
+            .to receive(:check_replica_status?)
+            .and_raise(PG::Error)
+
+          expect(host).not_to be_online
+        end
+      end
+    end
   end
 
   describe '#refresh_status' do