Gracefully handle case when Geo secondary does not have the right db_key_base

If the admin uses the wrong db_key_base on the secondary, an obscure OpenSSL
error shows up on the secondary and results in an Error 500. This change
helps make it more obvious to admins what needs to be fixed.

Closes #2181

changelogs/unreleased-ee/sh-geo-handle-invalid-db-key-base.yml

+---
+title: Gracefully handle case when Geo secondary does not have the right db_key_base
+merge_request:
+author:

lib/api/geo.rb

@@ -94,8 +94,12 @@ module API
       def authenticate_by_gitlab_geo_node_token!
         auth_header = headers['Authorization']
 
-        unless auth_header && Gitlab::Geo::JwtRequestDecoder.new(auth_header).decode
-          unauthorized!
+        begin
+          unless auth_header && Gitlab::Geo::JwtRequestDecoder.new(auth_header).decode
+            unauthorized!
+          end
+        rescue Gitlab::Geo::InvalidDecryptionKeyError => e
+          render_api_error!(e.to_s, 401)
         end
       end
 

lib/gitlab/geo/jwt_request_decoder.rb

 module Gitlab
   module Geo
+    InvalidDecryptionKeyError = Class.new(StandardError)
+
     class JwtRequestDecoder
       IAT_LEEWAY = 60.seconds.to_i
 
@@ -22,7 +24,13 @@ module Gitlab
         # For example:
         # JWT payload = { "data": { "oid": "12345" }, iat: 123456 }
         #
-        data = decode_auth_header
+        begin
+          data = decode_auth_header
+        rescue OpenSSL::Cipher::CipherError
+          message = 'Error decrypting the Geo secret from the database. Check that the primary and secondary have the same db_key_base.'
+          Rails.logger.error(message)
+          raise InvalidDecryptionKeyError.new(message)
+        end
 
         return unless data.present?
 

spec/lib/gitlab/geo/jwt_request_decoder_spec.rb

@@ -32,5 +32,11 @@ describe Gitlab::Geo::JwtRequestDecoder do
 
       expect(subject.decode).to be_nil
     end
+
+    it 'raises invalid decryption key error' do
+      allow_any_instance_of(described_class).to receive(:decode_auth_header).and_raise(Gitlab::Geo::InvalidDecryptionKeyError)
+
+      expect { subject.decode }.to raise_error(Gitlab::Geo::InvalidDecryptionKeyError)
+    end
   end
 end

spec/requests/api/geo_spec.rb

@@ -287,6 +287,14 @@ describe API::Geo, api: true do
       expect(response).to have_http_status(401)
     end
 
+    it 'responds with 401 when the db_key_base is wrong' do
+      allow_any_instance_of(Gitlab::Geo::JwtRequestDecoder).to receive(:decode).and_raise(Gitlab::Geo::InvalidDecryptionKeyError)
+
+      get api('/geo/status'), nil, request.headers
+
+      expect(response).to have_http_status(401)
+    end
+
     context 'when requesting secondary node with valid auth header' do
       before(:each) do
         allow(Gitlab::Geo).to receive(:current_node) { secondary_node }