Merge branch '208250-collect-object-store-config-in-usage-data' into 'master'

Collect object store config in usage data

Closes #208250

See merge request gitlab-org/gitlab!29149

changelogs/unreleased/208250-collect-object-store-config-in-usage-data.yml

+---
+title: Collect object store config in usage data
+merge_request: 29149
+author:
+type: added

lib/gitlab/usage_data.rb

@@ -24,6 +24,7 @@ module Gitlab
           .merge(features_usage_data)
           .merge(components_usage_data)
           .merge(cycle_analytics_usage_data)
+          .merge(object_store_usage_data)
       end
 
       def to_json(force_refresh: false)
@@ -237,6 +238,40 @@ module Gitlab
         'unknown_app_server_type'
       end
 
+      def object_store_config(component)
+        config = alt_usage_data(fallback: nil) do
+          Settings[component]['object_store']
+        end
+
+        if config
+          {
+            enabled: alt_usage_data { Settings[component]['enabled'] },
+            object_store: {
+              enabled: alt_usage_data { config['enabled'] },
+              direct_upload: alt_usage_data { config['direct_upload'] },
+              background_upload: alt_usage_data { config['background_upload'] },
+              provider: alt_usage_data { config['connection']['provider'] }
+            }
+          }
+        else
+          {
+            enabled: alt_usage_data { Settings[component]['enabled'] }
+          }
+        end
+      end
+
+      def object_store_usage_data
+        {
+          object_store: {
+            artifacts: object_store_config('artifacts'),
+            external_diffs: object_store_config('external_diffs'),
+            lfs: object_store_config('lfs'),
+            uploads: object_store_config('uploads'),
+            packages: object_store_config('packages')
+          }
+        }
+      end
+
       def ingress_modsecurity_usage
         ::Clusters::Applications::IngressModsecurityUsageService.new.execute
       end

spec/lib/gitlab/usage_data_spec.rb

@@ -7,6 +7,8 @@ describe Gitlab::UsageData, :aggregate_failures do
 
   before do
     allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
+
+    stub_object_store_settings
   end
 
   shared_examples "usage data execution" do
@@ -82,6 +84,16 @@ describe Gitlab::UsageData, :aggregate_failures do
         expect(count_data[:clusters_management_project]).to eq(1)
       end
 
+      it 'gathers object store usage correctly' do
+        expect(subject[:object_store]).to eq(
+          { artifacts: { enabled: true, object_store: { enabled: true, direct_upload: true, background_upload: false, provider: "AWS" } },
+           external_diffs: { enabled: false },
+           lfs: { enabled: true, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } },
+           uploads: { enabled: nil, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } },
+           packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: true, provider: "AWS" } } }
+        )
+      end
+
       it 'works when queries time out' do
         allow_any_instance_of(ActiveRecord::Relation)
           .to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
@@ -223,6 +235,66 @@ describe Gitlab::UsageData, :aggregate_failures do
         end
       end
 
+      describe '#object_store_config' do
+        let(:component) { 'lfs' }
+
+        subject { described_class.object_store_config(component) }
+
+        context 'when object_store is not configured' do
+          it 'returns component enable status only' do
+            allow(Settings).to receive(:[]).with(component).and_return({ 'enabled' => false })
+
+            expect(subject).to eq({ enabled: false })
+          end
+        end
+
+        context 'when object_store is configured' do
+          it 'returns filtered object store config' do
+            allow(Settings).to receive(:[]).with(component)
+              .and_return(
+                { 'enabled' => true,
+                  'object_store' =>
+                  { 'enabled' => true,
+                    'remote_directory' => component,
+                    'direct_upload' => true,
+                    'connection' =>
+                  { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
+                    'background_upload' => false,
+                    'proxy_download' => false } })
+
+            expect(subject).to eq(
+              { enabled: true, object_store: { enabled: true, direct_upload: true, background_upload: false, provider: "AWS" } })
+          end
+        end
+
+        context 'when retrieve component setting meets exception' do
+          it 'returns -1 for component enable status' do
+            allow(Settings).to receive(:[]).with(component).and_raise(StandardError)
+
+            expect(subject).to eq({ enabled: -1 })
+          end
+        end
+      end
+
+      describe '#object_store_usage_data' do
+        subject { described_class.object_store_usage_data }
+
+        it 'fetches object store config of five components' do
+          %w(artifacts external_diffs lfs uploads packages).each do |component|
+            expect(described_class).to receive(:object_store_config).with(component).and_return("#{component}_object_store_config")
+          end
+
+          expect(subject).to eq(
+            object_store: {
+              artifacts: 'artifacts_object_store_config',
+              external_diffs: 'external_diffs_object_store_config',
+              lfs: 'lfs_object_store_config',
+              uploads: 'uploads_object_store_config',
+              packages: 'packages_object_store_config'
+            })
+        end
+      end
+
       describe '#cycle_analytics_usage_data' do
         subject { described_class.cycle_analytics_usage_data }
 

spec/support/helpers/usage_data_helpers.rb

@@ -153,5 +153,59 @@ module UsageDataHelpers
       projects_with_expiration_policy_enabled_with_older_than_set_to_14d
       projects_with_expiration_policy_enabled_with_older_than_set_to_30d
       projects_with_expiration_policy_enabled_with_older_than_set_to_90d
+      object_store
     ).freeze
+
+  def stub_object_store_settings
+    allow(Settings).to receive(:[]).with('artifacts')
+      .and_return(
+        { 'enabled' => true,
+         'object_store' =>
+         { 'enabled' => true,
+          'remote_directory' => 'artifacts',
+          'direct_upload' => true,
+          'connection' =>
+         { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
+           'background_upload' => false,
+           'proxy_download' => false } }
+      )
+
+    allow(Settings).to receive(:[]).with('external_diffs').and_return({ 'enabled' => false })
+
+    allow(Settings).to receive(:[]).with('lfs')
+      .and_return(
+        { 'enabled' => true,
+         'object_store' =>
+         { 'enabled' => false,
+          'remote_directory' => 'lfs-objects',
+          'direct_upload' => true,
+          'connection' =>
+         { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
+           'background_upload' => false,
+           'proxy_download' => false } }
+      )
+    allow(Settings).to receive(:[]).with('uploads')
+      .and_return(
+        { 'object_store' =>
+          { 'enabled' => false,
+          'remote_directory' => 'uploads',
+          'direct_upload' => true,
+          'connection' =>
+          { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
+           'background_upload' => false,
+           'proxy_download' => false } }
+      )
+    allow(Settings).to receive(:[]).with('packages')
+      .and_return(
+        { 'enabled' => true,
+         'object_store' =>
+         { 'enabled' => false,
+          'remote_directory' => 'packages',
+          'direct_upload' => false,
+          'connection' =>
+         { 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
+           'background_upload' => true,
+           'proxy_download' => false } }
+      )
+  end
 end