Commit e1a8aa81 authored by Qingyu Zhao's avatar Qingyu Zhao

Collect object store config in usage data

parent 593e74a8
---
title: Collect object store config in usage data
merge_request: 29149
author:
type: added
......@@ -24,6 +24,7 @@ module Gitlab
.merge(features_usage_data)
.merge(components_usage_data)
.merge(cycle_analytics_usage_data)
.merge(object_store_usage_data)
end
def to_json(force_refresh: false)
......@@ -237,6 +238,40 @@ module Gitlab
'unknown_app_server_type'
end
def object_store_config(component)
config = alt_usage_data(fallback: nil) do
Settings[component]['object_store']
end
if config
{
enabled: alt_usage_data { Settings[component]['enabled'] },
object_store: {
enabled: alt_usage_data { config['enabled'] },
direct_upload: alt_usage_data { config['direct_upload'] },
background_upload: alt_usage_data { config['background_upload'] },
provider: alt_usage_data { config['connection']['provider'] }
}
}
else
{
enabled: alt_usage_data { Settings[component]['enabled'] }
}
end
end
def object_store_usage_data
{
object_store: {
artifacts: object_store_config('artifacts'),
external_diffs: object_store_config('external_diffs'),
lfs: object_store_config('lfs'),
uploads: object_store_config('uploads'),
packages: object_store_config('packages')
}
}
end
def ingress_modsecurity_usage
::Clusters::Applications::IngressModsecurityUsageService.new.execute
end
......
......@@ -7,6 +7,8 @@ describe Gitlab::UsageData, :aggregate_failures do
before do
allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
stub_object_store_settings
end
shared_examples "usage data execution" do
......@@ -82,6 +84,16 @@ describe Gitlab::UsageData, :aggregate_failures do
expect(count_data[:clusters_management_project]).to eq(1)
end
it 'gathers object store usage correctly' do
expect(subject[:object_store]).to eq(
{ artifacts: { enabled: true, object_store: { enabled: true, direct_upload: true, background_upload: false, provider: "AWS" } },
external_diffs: { enabled: false },
lfs: { enabled: true, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } },
uploads: { enabled: nil, object_store: { enabled: false, direct_upload: true, background_upload: false, provider: "AWS" } },
packages: { enabled: true, object_store: { enabled: false, direct_upload: false, background_upload: true, provider: "AWS" } } }
)
end
it 'works when queries time out' do
allow_any_instance_of(ActiveRecord::Relation)
.to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
......@@ -223,6 +235,66 @@ describe Gitlab::UsageData, :aggregate_failures do
end
end
describe '#object_store_config' do
let(:component) { 'lfs' }
subject { described_class.object_store_config(component) }
context 'when object_store is not configured' do
it 'returns component enable status only' do
allow(Settings).to receive(:[]).with(component).and_return({ 'enabled' => false })
expect(subject).to eq({ enabled: false })
end
end
context 'when object_store is configured' do
it 'returns filtered object store config' do
allow(Settings).to receive(:[]).with(component)
.and_return(
{ 'enabled' => true,
'object_store' =>
{ 'enabled' => true,
'remote_directory' => component,
'direct_upload' => true,
'connection' =>
{ 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
'background_upload' => false,
'proxy_download' => false } })
expect(subject).to eq(
{ enabled: true, object_store: { enabled: true, direct_upload: true, background_upload: false, provider: "AWS" } })
end
end
context 'when retrieve component setting meets exception' do
it 'returns -1 for component enable status' do
allow(Settings).to receive(:[]).with(component).and_raise(StandardError)
expect(subject).to eq({ enabled: -1 })
end
end
end
describe '#object_store_usage_data' do
subject { described_class.object_store_usage_data }
it 'fetches object store config of five components' do
%w(artifacts external_diffs lfs uploads packages).each do |component|
expect(described_class).to receive(:object_store_config).with(component).and_return("#{component}_object_store_config")
end
expect(subject).to eq(
object_store: {
artifacts: 'artifacts_object_store_config',
external_diffs: 'external_diffs_object_store_config',
lfs: 'lfs_object_store_config',
uploads: 'uploads_object_store_config',
packages: 'packages_object_store_config'
})
end
end
describe '#cycle_analytics_usage_data' do
subject { described_class.cycle_analytics_usage_data }
......
......@@ -153,5 +153,59 @@ module UsageDataHelpers
projects_with_expiration_policy_enabled_with_older_than_set_to_14d
projects_with_expiration_policy_enabled_with_older_than_set_to_30d
projects_with_expiration_policy_enabled_with_older_than_set_to_90d
object_store
).freeze
def stub_object_store_settings
allow(Settings).to receive(:[]).with('artifacts')
.and_return(
{ 'enabled' => true,
'object_store' =>
{ 'enabled' => true,
'remote_directory' => 'artifacts',
'direct_upload' => true,
'connection' =>
{ 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
'background_upload' => false,
'proxy_download' => false } }
)
allow(Settings).to receive(:[]).with('external_diffs').and_return({ 'enabled' => false })
allow(Settings).to receive(:[]).with('lfs')
.and_return(
{ 'enabled' => true,
'object_store' =>
{ 'enabled' => false,
'remote_directory' => 'lfs-objects',
'direct_upload' => true,
'connection' =>
{ 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
'background_upload' => false,
'proxy_download' => false } }
)
allow(Settings).to receive(:[]).with('uploads')
.and_return(
{ 'object_store' =>
{ 'enabled' => false,
'remote_directory' => 'uploads',
'direct_upload' => true,
'connection' =>
{ 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
'background_upload' => false,
'proxy_download' => false } }
)
allow(Settings).to receive(:[]).with('packages')
.and_return(
{ 'enabled' => true,
'object_store' =>
{ 'enabled' => false,
'remote_directory' => 'packages',
'direct_upload' => false,
'connection' =>
{ 'provider' => 'AWS', 'aws_access_key_id' => 'minio', 'aws_secret_access_key' => 'gdk-minio', 'region' => 'gdk', 'endpoint' => 'http://127.0.0.1:9000', 'path_style' => true },
'background_upload' => true,
'proxy_download' => false } }
)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment