Merge branch 'ak/update-google-auth' into 'master'

Support IAP protected prometheus installations

See merge request gitlab-org/gitlab!33508

Gemfile

@@ -112,14 +112,14 @@ gem 'fog-aws', '~> 3.5'
 # Locked until fog-google resolves https://github.com/fog/fog-google/issues/421.
 # Also see config/initializers/fog_core_patch.rb.
 gem 'fog-core', '= 2.1.0'
-gem 'fog-google', '~> 1.9'
+gem 'fog-google', '~> 1.10'
 gem 'fog-local', '~> 0.6'
 gem 'fog-openstack', '~> 1.0'
 gem 'fog-rackspace', '~> 0.1.1'
 gem 'fog-aliyun', '~> 0.3'
 
 # for Google storage
-gem 'google-api-client', '~> 0.23'
+gem 'google-api-client', '~> 0.33'
 
 # for aws storage
 gem 'unf', '~> 0.1.4'

Gemfile.lock

@@ -286,7 +286,7 @@ GEM
     factory_bot_rails (5.1.0)
       factory_bot (~> 5.1.0)
       railties (>= 4.2.0)
-    faraday (0.15.4)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     faraday-http-cache (2.0.0)
       faraday (~> 0.8)
@@ -330,11 +330,11 @@ GEM
       excon (~> 0.58)
       formatador (~> 0.2)
       mime-types
-    fog-google (1.9.1)
+    fog-google (1.10.0)
       fog-core (<= 2.1.0)
       fog-json (~> 1.2)
       fog-xml (~> 0.1.0)
-      google-api-client (~> 0.23.0)
+      google-api-client (>= 0.32, < 0.34)
     fog-json (1.2.0)
       fog-core
       multi_json (~> 1.10)
@@ -419,23 +419,24 @@ GEM
       actionpack (>= 3.0)
       multi_json
       request_store (>= 1.0)
-    google-api-client (0.23.4)
+    google-api-client (0.33.2)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (>= 0.5, < 0.7.0)
+      googleauth (~> 0.9)
       httpclient (>= 2.8.1, < 3.0)
-      mime-types (~> 3.0)
+      mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
+      signet (~> 0.12)
     google-protobuf (3.8.0)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
-    googleauth (0.6.6)
-      faraday (~> 0.12)
+    googleauth (0.12.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
-      memoist (~> 0.12)
+      memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
+      signet (~> 0.14)
     gpgme (2.0.20)
       mini_portile2 (~> 2.3)
     grape (1.1.0)
@@ -1010,9 +1011,9 @@ GEM
     sidekiq-cron (1.0.4)
       fugit (~> 1.1)
       sidekiq (>= 4.2.1)
-    signet (0.11.0)
+    signet (0.14.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     simple_po_parser (1.1.2)
@@ -1224,7 +1225,7 @@ DEPENDENCIES
   fog-aliyun (~> 0.3)
   fog-aws (~> 3.5)
   fog-core (= 2.1.0)
-  fog-google (~> 1.9)
+  fog-google (~> 1.10)
   fog-local (~> 0.6)
   fog-openstack (~> 1.0)
   fog-rackspace (~> 0.1.1)
@@ -1250,7 +1251,7 @@ DEPENDENCIES
   gitlab_chronic_duration (~> 0.10.6.2)
   gitlab_omniauth-ldap (~> 2.1.1)
   gon (~> 6.2)
-  google-api-client (~> 0.23)
+  google-api-client (~> 0.33)
   google-protobuf (~> 3.8.0)
   gpgme (~> 2.0.19)
   grape (~> 1.1.0)

app/controllers/concerns/service_params.rb

@@ -28,6 +28,8 @@ module ServiceParams
     :drone_url,
     :enable_ssl_verification,
     :external_wiki_url,
+    :google_iap_service_account_json,
+    :google_iap_audience_client_id,
     # We're using `issues_events` and `merge_requests_events`
     # in the view so we still need to explicitly state them
     # here. `Service#event_names` would only give

app/models/project_services/prometheus_service.rb

@@ -5,6 +5,8 @@ class PrometheusService < MonitoringService
 
   #  Access to prometheus is directly through the API
   prop_accessor :api_url
+  prop_accessor :google_iap_service_account_json
+  prop_accessor :google_iap_audience_client_id
   boolean_accessor :manual_configuration
 
   # We need to allow the self-monitoring project to connect to the internal
@@ -49,7 +51,7 @@ class PrometheusService < MonitoringService
   end
 
   def fields
-    [
+    result = [
       {
         type: 'checkbox',
         name: 'manual_configuration',
@@ -64,6 +66,27 @@ class PrometheusService < MonitoringService
         required: true
       }
     ]
+
+    if Feature.enabled?(:prometheus_service_iap_auth)
+      result += [
+        {
+          type: 'text',
+          name: 'google_iap_audience_client_id',
+          title: 'Google IAP Audience Client ID',
+          placeholder: s_('PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'),
+          required: false
+        },
+        {
+          type: 'textarea',
+          name: 'google_iap_service_account_json',
+          title: 'Google IAP Service Account JSON',
+          placeholder: s_('PrometheusService|Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'),
+          required: false
+        }
+      ]
+    end
+
+    result
   end
 
   # Check we can connect to the Prometheus API
@@ -77,7 +100,14 @@ class PrometheusService < MonitoringService
   def prometheus_client
     return unless should_return_client?
 
-    Gitlab::PrometheusClient.new(api_url, allow_local_requests: allow_local_api_url?)
+    options = { allow_local_requests: allow_local_api_url? }
+
+    if Feature.enabled?(:prometheus_service_iap_auth) && behind_iap?
+      # Adds the Authorization header
+      options[:headers] = iap_client.apply({})
+    end
+
+    Gitlab::PrometheusClient.new(api_url, options)
   end
 
   def prometheus_available?
@@ -149,4 +179,12 @@ class PrometheusService < MonitoringService
 
     Prometheus::CreateDefaultAlertsWorker.perform_async(project_id)
   end
+
+  def behind_iap?
+    manual_configuration? && google_iap_audience_client_id.present? && google_iap_service_account_json.present?
+  end
+
+  def iap_client
+    @iap_client ||= Google::Auth::Credentials.new(Gitlab::Json.parse(google_iap_service_account_json), target_audience: google_iap_audience_client_id).client
+  end
 end

changelogs/unreleased/ak-update-google-auth.yml

+---
+title: Support IAP protected prometheus installations
+merge_request: 33508
+author:
+type: added

config/initializers/google_api_client.rb

 # frozen_string_literal: true
-#
-# google-api-client >= 0.26.0 supports enabling CloudRun and Istio during
-# cluster creation, but fog-google currently hard deps on '~> 0.23.0', which
-# prevents us from upgrading. We are injecting these options as hashes below
-# as a workaround until this is resolved.
-#
-# This can be removed once fog-google and google-api-client can be upgraded.
-# See https://gitlab.com/gitlab-org/gitlab/issues/31280 for more details.
-#
 
-require 'google/apis/container_v1beta1'
 require 'google/apis/options'
 
+# these require solve load order issues (undefined constant Google::Apis::ServerError and Signet::RemoteServerError, rescued in multiple places)
+require 'google/apis/errors'
+require 'signet/errors'
+
 # As stated in https://github.com/googleapis/google-api-ruby-client#errors--retries,
 # enabling retries is strongly encouraged but disabled by default. Large uploads
 # that may hit timeouts will mainly benefit from this.
 Google::Apis::RequestOptions.default.retries = 3 if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_GOOGLE_API_RETRIES', true))
-
-Google::Apis::ContainerV1beta1::AddonsConfig::Representation.tap do |representation|
-  representation.hash :cloud_run_config, as: 'cloudRunConfig'
-  representation.hash :istio_config, as: 'istioConfig'
-end

doc/api/services.md

@@ -1007,6 +1007,8 @@ Parameters:
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
 | `api_url` | string | true | Prometheus API Base URL. For example, `http://prometheus.example.com/`. |
+| `google_iap_audience_client_id` | string | false | Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com) |
+| `google_iap_service_account_json` | string | false | credentials.json file for your service account, like { "type": "service_account", "project_id": ... } |
 
 ### Delete Prometheus service
 

lib/api/helpers/services_helpers.rb

@@ -583,6 +583,18 @@ module API
               name: :api_url,
               type: String,
               desc: 'Prometheus API Base URL, like http://prometheus.example.com/'
+            },
+            {
+              required: true,
+              name: :google_iap_audience_client_id,
+              type: String,
+              desc: 'Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'
+            },
+            {
+              required: true,
+              name: :google_iap_service_account_json,
+              type: String,
+              desc: 'Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'
             }
           ],
           'pushover' => [

locale/gitlab.pot

@@ -17475,12 +17475,18 @@ msgstr ""
 msgid "PrometheusService|Automatically deploy and configure Prometheus on your clusters to monitor your project’s environments"
 msgstr ""
 
+msgid "PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)"
+msgstr ""
+
 msgid "PrometheusService|Common metrics"
 msgstr ""
 
 msgid "PrometheusService|Common metrics are automatically monitored based on a library of metrics from popular exporters."
 msgstr ""
 
+msgid "PrometheusService|Contents of the credentials.json file of your service account, like: { \"type\": \"service_account\", \"project_id\": ... }"
+msgstr ""
+
 msgid "PrometheusService|Custom metrics"
 msgstr ""
 

spec/initializers/google_api_client_spec.rb

-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe './config/initializers/google_api_client.rb' do
-  subject { Google::Apis::ContainerV1beta1 }
-
-  it 'is needed' do |example|
-    is_expected.not_to be_const_defined(:CloudRunConfig),
-      <<-MSG.strip_heredoc
-        The google-api-client gem has been upgraded!
-        Remove:
-          #{example.example_group.description}
-          #{example.file_path}
-      MSG
-  end
-end

spec/models/project_services/prometheus_service_spec.rb

@@ -252,6 +252,26 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
         end
       end
     end
+
+    context 'behind IAP' do
+      let(:manual_configuration) { true }
+
+      before do
+        # dummy private key generated only for this test to pass openssl validation
+        service.google_iap_service_account_json = '{"type":"service_account","private_key":"-----BEGIN RSA PRIVATE KEY-----\nMIIBOAIBAAJAU85LgUY5o6j6j/07GMLCNUcWJOBA1buZnNgKELayA6mSsHrIv31J\nY8kS+9WzGPQninea7DcM4hHA7smMgQD1BwIDAQABAkAqKxMy6PL3tn7dFL43p0ex\nJyOtSmlVIiAZG1t1LXhE/uoLpYi5DnbYqGgu0oih+7nzLY/dXpNpXUmiRMOUEKmB\nAiEAoTi2rBXbrLSi2C+H7M/nTOjMQQDuZ8Wr4uWpKcjYJTMCIQCFEskL565oFl/7\nRRQVH+cARrAsAAoJSbrOBAvYZ0PI3QIgIEFwis10vgEF86rOzxppdIG/G+JL0IdD\n9IluZuXAGPECIGUo7qSaLr75o2VEEgwtAFH5aptIPFjrL5LFCKwtdB4RAiAYZgFV\nHCMmaooAw/eELuMoMWNYmujZ7VaAnOewGDW0uw==\n-----END RSA PRIVATE KEY-----\n"}'
+        service.google_iap_audience_client_id = "IAP_CLIENT_ID.apps.googleusercontent.com"
+
+        stub_request(:post, "https://oauth2.googleapis.com/token").to_return(status: 200, body: '{"id_token": "FOO"}', headers: { 'Content-Type': 'application/json; charset=UTF-8' })
+
+        stub_feature_flags(prometheus_service_iap_auth: true)
+      end
+
+      it 'includes the authorization header' do
+        expect(service.prometheus_client).not_to be_nil
+        expect(service.prometheus_client.send(:options)).to have_key(:headers)
+        expect(service.prometheus_client.send(:options)[:headers]).to eq(authorization: "Bearer FOO")
+      end
+    end
   end
 
   describe '#prometheus_available?' do
@@ -457,9 +477,33 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
         }
       ]
     end
+    let(:feature_flagged_fields) do
+      [
+        {
+          type: 'text',
+          name: 'google_iap_audience_client_id',
+          title: 'Google IAP Audience Client ID',
+          placeholder: s_('PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'),
+          required: false
+        },
+        {
+          type: 'textarea',
+          name: 'google_iap_service_account_json',
+          title: 'Google IAP Service Account JSON',
+          placeholder: s_('PrometheusService|Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'),
+          required: false
+        }
+      ]
+    end
 
     it 'returns fields' do
+      stub_feature_flags(prometheus_service_iap_auth: false)
       expect(service.fields).to eq(expected_fields)
     end
+
+    it 'returns fields with feature flag on' do
+      stub_feature_flags(prometheus_service_iap_auth: true)
+      expect(service.fields).to eq(expected_fields + feature_flagged_fields)
+    end
   end
 end