Commit 840e8556 authored by Gilbert Roulot's avatar Gilbert Roulot Committed by Robert Speicher

Usage data for security products

parent 9b649a36
......@@ -430,6 +430,7 @@ ActiveRecord::Schema.define(version: 20180803001726) do
add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
add_index "ci_builds", ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))", using: :btree
add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
......
......@@ -33,7 +33,8 @@ disable the version check at **Admin area > Settings > Usage statistics**.
> [Introduced][ee-557] in GitLab Enterprise Edition 8.10. More statistics
[were added][ee-735] in GitLab Enterprise Edition
8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1.
8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1. More statistics
[were added][[ee-6602] in 11.2
GitLab sends a weekly payload containing usage data to GitLab Inc. The usage
ping uses high-level data to help our product, support, and sales teams. It does
......@@ -70,3 +71,4 @@ production: &base
[ee-557]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/557
[ee-735]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/735
[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361
[ee-6602]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/6602
......@@ -307,6 +307,19 @@ class License < ActiveRecord::Base
restricted_attr(:plan).presence || STARTER_PLAN
end
def edition
case restricted_attr(:plan)
when 'ultimate'
'EEU'
when 'premium'
'EEP'
when 'starter'
'EES'
else # Older licenses
'EE'
end
end
def current_active_users_count
@current_active_users_count ||= begin
if exclude_guests_from_active_count?
......
---
title: Add security products to usage ping
merge_request: 6602
author:
type: changed
# frozen_string_literal: true
class AddNameIndexToCiBuilds < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_builds,
[:name],
name: 'index_ci_builds_on_name_for_security_products_values',
where: "name IN ('container_scanning','dast','dependency_scanning','license_management','sast')"
end
def down
remove_concurrent_index :ci_builds, [:name], name: 'index_ci_builds_on_name_for_security_products_values'
end
end
# frozen_string_literal: true
module EE
module Gitlab
module UsageData
extend ::Gitlab::Utils::Override
override :features_usage_data
def features_usage_data
super.merge(features_usage_data_ee)
end
def features_usage_data_ee
{
elasticsearch_enabled: ::Gitlab::CurrentSettings.elasticsearch_search?,
geo_enabled: ::Gitlab::Geo.enabled?
}
end
override :license_usage_data
def license_usage_data
usage_data = super
license = ::License.current
usage_data[:edition] =
if license
license.edition
else
'EE Free'
end
if license
usage_data[:license_md5] = license.md5
usage_data[:license_id] = license.license_id
usage_data[:historical_max_users] = ::HistoricalData.max_historical_user_count
usage_data[:licensee] = license.licensee
usage_data[:license_user_count] = license.restricted_user_count
usage_data[:license_starts_at] = license.starts_at
usage_data[:license_expires_at] = license.expires_at
usage_data[:license_plan] = license.plan
usage_data[:license_add_ons] = license.add_ons
usage_data[:license_trial] = license.trial?
end
usage_data
end
def projects_mirrored_with_pipelines_enabled
::Project.joins(:project_feature).where(
mirror: true,
mirror_trigger_builds: true,
project_features: {
builds_access_level: ::ProjectFeature::ENABLED
}
).count
end
def service_desk_counts
return {} unless ::License.feature_available?(:service_desk)
projects_with_service_desk = ::Project.where(service_desk_enabled: true)
{
service_desk_enabled_projects: projects_with_service_desk.count,
service_desk_issues: ::Issue.where(
project: projects_with_service_desk,
author: ::User.support_bot,
confidential: true
).count
}
end
def security_products_usage
types = {
container_scanning: :container_scanning_jobs,
dast: :dast_jobs,
dependency_scanning: :dependency_scanning_jobs,
license_management: :license_management_jobs,
sast: :sast_jobs
}
results = ::Ci::Build.where(name: types.keys).group(:name).count
results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value }
end
override :system_usage_data
def system_usage_data
usage_data = super
usage_data[:counts] = usage_data[:counts].merge({
epics: ::Epic.count,
geo_nodes: ::GeoNode.count,
ldap_group_links: ::LdapGroupLink.count,
ldap_keys: ::LDAPKey.count,
ldap_users: ::User.ldap.count,
projects_reporting_ci_cd_back_to_github: ::GithubService.without_defaults.active.count,
projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled
}).merge(service_desk_counts).merge(security_products_usage)
usage_data
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::UsageData do
let(:projects) { create_list(:project, 3) }
let!(:board) { create(:board, project: projects[0]) }
describe '#data' do
before do
pipeline = create(:ci_pipeline, project: projects[0])
create(:ci_build, name: 'container_scanning', pipeline: pipeline)
create(:ci_build, name: 'dast', pipeline: pipeline)
create(:ci_build, name: 'dependency_scanning', pipeline: pipeline)
create(:ci_build, name: 'license_management', pipeline: pipeline)
create(:ci_build, name: 'sast', pipeline: pipeline)
end
subject { described_class.data }
it "gathers usage data" do
expect(subject.keys).to include(*%i(
historical_max_users
license_add_ons
license_plan
license_expires_at
license_starts_at
license_user_count
license_trial
licensee
license_md5
license_id
elasticsearch_enabled
geo_enabled
))
end
it "gathers usage counts" do
count_data = subject[:counts]
expect(count_data[:boards]).to eq(1)
expect(count_data[:projects]).to eq(3)
expect(count_data.keys).to include(*%i(
projects_mirrored_with_pipelines_enabled
epics
geo_nodes
ldap_group_links
ldap_keys
ldap_users
projects_reporting_ci_cd_back_to_github
container_scanning_jobs
dast_jobs
dependency_scanning_jobs
license_management_jobs
sast_jobs
))
end
it 'gathers security products usage data' do
count_data = subject[:counts]
expect(count_data[:container_scanning_jobs]).to eq(1)
expect(count_data[:dast_jobs]).to eq(1)
expect(count_data[:dependency_scanning_jobs]).to eq(1)
expect(count_data[:license_management_jobs]).to eq(1)
expect(count_data[:sast_jobs]).to eq(1)
end
end
describe '#features_usage_data_ee' do
subject { described_class.features_usage_data_ee }
it 'gathers feature usage data of EE' do
expect(subject[:elasticsearch_enabled]).to eq(Gitlab::CurrentSettings.elasticsearch_search?)
expect(subject[:geo_enabled]).to eq(Gitlab::Geo.enabled?)
end
end
describe 'License edition names' do
let(:ultimate) { create(:license, plan: 'ultimate') }
let(:premium) { create(:license, plan: 'premium') }
let(:starter) { create(:license, plan: 'starter') }
let(:old) { create(:license, plan: 'other') }
it "have expected values" do
expect(ultimate.edition).to eq('EEU')
expect(premium.edition).to eq('EEP')
expect(starter.edition).to eq('EES')
expect(old.edition).to eq('EE')
end
end
describe '#license_usage_data' do
subject { described_class.license_usage_data }
it "gathers license data" do
license = ::License.current
expect(subject[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
expect(subject[:license_id]).to eq(license.license_id)
expect(subject[:historical_max_users]).to eq(::HistoricalData.max_historical_user_count)
expect(subject[:licensee]).to eq(license.licensee)
expect(subject[:license_user_count]).to eq(license.restricted_user_count)
expect(subject[:license_starts_at]).to eq(license.starts_at)
expect(subject[:license_expires_at]).to eq(license.expires_at)
expect(subject[:license_add_ons]).to eq(license.add_ons)
expect(subject[:license_trial]).to eq(license.trial?)
end
end
describe '.service_desk_counts' do
subject { described_class.service_desk_counts }
before do
Project.update_all(service_desk_enabled: true)
end
context 'when Service Desk is disabled' do
it 'returns an empty hash' do
stub_licensed_features(service_desk: false)
expect(subject).to eq({})
end
end
context 'when there is no license' do
it 'returns an empty hash' do
allow(License).to receive(:current).and_return(nil)
expect(subject).to eq({})
end
end
context 'when Service Desk is enabled' do
it 'gathers Service Desk data' do
create_list(:issue, 3, confidential: true, author: User.support_bot, project: projects[0])
stub_licensed_features(service_desk: true)
allow(::EE::Gitlab::ServiceDesk).to receive(:enabled?).with(anything).and_return(true)
expect(subject).to eq(service_desk_enabled_projects: 3,
service_desk_issues: 3)
end
end
end
end
module Gitlab
class UsageData
class << self
prepend EE::Gitlab::UsageData
def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
end
......@@ -27,23 +29,6 @@ module Gitlab
edition: 'EE'
}
license = ::License.current
usage_data[:edition] = license_edition(license)
if license
usage_data[:license_md5] = license.md5
usage_data[:license_id] = license.license_id
usage_data[:historical_max_users] = ::HistoricalData.max_historical_user_count
usage_data[:licensee] = license.licensee
usage_data[:license_user_count] = license.restricted_user_count
usage_data[:license_starts_at] = license.starts_at
usage_data[:license_expires_at] = license.expires_at
usage_data[:license_plan] = license.plan
usage_data[:license_add_ons] = license.add_ons
usage_data[:license_trial] = license.trial?
end
usage_data
end
......@@ -65,8 +50,6 @@ module Gitlab
deploy_keys: DeployKey.count,
deployments: Deployment.count,
environments: ::Environment.count,
epics: ::Epic.count,
geo_nodes: GeoNode.count,
clusters: ::Clusters::Cluster.count,
clusters_enabled: ::Clusters::Cluster.enabled.count,
clusters_disabled: ::Clusters::Cluster.disabled.count,
......@@ -81,9 +64,6 @@ module Gitlab
issues: Issue.count,
keys: Key.count,
labels: Label.count,
ldap_group_links: LdapGroupLink.count,
ldap_keys: LDAPKey.count,
ldap_users: User.ldap.count,
lfs_objects: LfsObject.count,
merge_requests: MergeRequest.count,
milestones: Milestone.count,
......@@ -91,8 +71,6 @@ module Gitlab
pages_domains: PagesDomain.count,
projects: Project.count,
projects_imported_from_github: Project.where(import_type: 'github').count,
projects_reporting_ci_cd_back_to_github: GithubService.without_defaults.active.count,
projects_mirrored_with_pipelines_enabled: projects_mirrored_with_pipelines_enabled,
protected_branches: ProtectedBranch.count,
releases: Release.count,
remote_mirrors: RemoteMirror.count,
......@@ -100,20 +78,7 @@ module Gitlab
todos: Todo.count,
uploads: Upload.count,
web_hooks: WebHook.count
}.merge(service_desk_counts).merge(services_usage)
}
end
def service_desk_counts
return {} unless ::License.feature_available?(:service_desk)
projects_with_service_desk = Project.where(service_desk_enabled: true)
{
service_desk_enabled_projects: projects_with_service_desk.count,
service_desk_issues: Issue.where(project: projects_with_service_desk,
author: User.support_bot,
confidential: true).count
}.merge(services_usage)
}
end
......@@ -122,7 +87,7 @@ module Gitlab
end
def features_usage_data
features_usage_data_ce.merge(features_usage_data_ee)
features_usage_data_ce
end
def features_usage_data_ce
......@@ -138,13 +103,6 @@ module Gitlab
}
end
def features_usage_data_ee
{
elasticsearch_enabled: Gitlab::CurrentSettings.elasticsearch_search?,
geo_enabled: Gitlab::Geo.enabled?
}
end
def components_usage_data
{
gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION },
......@@ -153,21 +111,6 @@ module Gitlab
}
end
def license_edition(license)
return 'EE Free' unless license
case license.plan
when 'ultimate'
'EEU'
when 'premium'
'EEP'
when 'starter'
'EES'
else # Older licenses
'EE'
end
end
def services_usage
types = {
JiraService: :projects_jira_active,
......@@ -179,16 +122,6 @@ module Gitlab
results = Service.unscoped.where(type: types.keys, active: true).group(:type).count
results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value }
end
def projects_mirrored_with_pipelines_enabled
Project.joins(:project_feature).where(
mirror: true,
mirror_trigger_builds: true,
project_features: {
builds_access_level: ProjectFeature::ENABLED
}
).count
end
end
end
end
......@@ -26,19 +26,9 @@ describe Gitlab::UsageData do
subject { described_class.data }
it "gathers usage data" do
expect(subject.keys).to match_array(%i(
expect(subject.keys).to include(*%i(
active_user_count
counts
historical_max_users
license_add_ons
license_plan
license_expires_at
license_starts_at
license_user_count
license_trial
licensee
license_md5
license_id
recorded_at
edition
version
......@@ -53,8 +43,6 @@ describe Gitlab::UsageData do
reply_by_email_enabled
container_registry_enabled
gitlab_shared_runners_enabled
elasticsearch_enabled
geo_enabled
gitlab_pages
git
database
......@@ -68,10 +56,9 @@ describe Gitlab::UsageData do
expect(count_data[:boards]).to eq(1)
expect(count_data[:projects]).to eq(3)
expect(count_data.keys).to match_array(%i(
expect(count_data.keys).to include(*%i(
boards
ci_builds
projects_mirrored_with_pipelines_enabled
ci_internal_pipelines
ci_external_pipelines
ci_pipeline_config_auto_devops
......@@ -84,8 +71,6 @@ describe Gitlab::UsageData do
deploy_keys
deployments
environments
epics
geo_nodes
clusters
clusters_enabled
clusters_disabled
......@@ -100,16 +85,12 @@ describe Gitlab::UsageData do
issues
keys
labels
ldap_group_links
ldap_keys
ldap_users
lfs_objects
merge_requests
milestones
notes
projects
projects_imported_from_github
projects_reporting_ci_cd_back_to_github
projects_jira_active
projects_slack_notifications_active
projects_slack_slash_active
......@@ -160,15 +141,6 @@ describe Gitlab::UsageData do
end
end
describe '#features_usage_data_ee' do
subject { described_class.features_usage_data_ee }
it 'gathers feature usage data of EE' do
expect(subject[:elasticsearch_enabled]).to eq(Gitlab::CurrentSettings.elasticsearch_search?)
expect(subject[:geo_enabled]).to eq(Gitlab::Geo.enabled?)
end
end
describe '#components_usage_data' do
subject { described_class.components_usage_data }
......@@ -185,58 +157,11 @@ describe Gitlab::UsageData do
subject { described_class.license_usage_data }
it "gathers license data" do
license = ::License.current
expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
expect(subject[:license_md5]).to eq(Digest::MD5.hexdigest(license.data))
expect(subject[:license_id]).to eq(license.license_id)
expect(subject[:version]).to eq(Gitlab::VERSION)
expect(subject[:licensee]).to eq(license.licensee)
expect(subject[:installation_type]).to eq(Gitlab::INSTALLATION_TYPE)
expect(subject[:active_user_count]).to eq(User.active.count)
expect(subject[:licensee]).to eq(license.licensee)
expect(subject[:license_user_count]).to eq(license.restricted_user_count)
expect(subject[:license_starts_at]).to eq(license.starts_at)
expect(subject[:license_expires_at]).to eq(license.expires_at)
expect(subject[:license_add_ons]).to eq(license.add_ons)
expect(subject[:license_trial]).to eq(license.trial?)
expect(subject[:recorded_at]).to be_a(Time)
end
end
describe '.service_desk_counts' do
subject { described_class.service_desk_counts }
before do
Project.update_all(service_desk_enabled: true)
end
context 'when Service Desk is disabled' do
it 'returns an empty hash' do
allow(License).to receive(:feature_available?).with(:service_desk).and_return(false)
expect(subject).to eq({})
end
end
context 'when there is no license' do
it 'returns an empty hash' do
allow(License).to receive(:current).and_return(nil)
expect(subject).to eq({})
end
end
context 'when Service Desk is enabled' do
it 'gathers Service Desk data' do
create_list(:issue, 3, confidential: true, author: User.support_bot, project: project)
allow(License).to receive(:feature_available?).with(:service_desk).and_return(true)
allow(::EE::Gitlab::ServiceDesk).to receive(:enabled?).with(anything).and_return(true)
expect(subject).to eq(service_desk_enabled_projects: 4,
service_desk_issues: 3)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment