Commit 5326031e authored by Douwe Maan's avatar Douwe Maan

Merge branch 'usage-count' into 'master'

Use approximate counts for big tables

Closes #51097

See merge request gitlab-org/gitlab-ce!23599
parents b4146c70 3720d02b
---
title: Use approximate count for big tables for usage statistics.
merge_request:
author:
type: fixed
...@@ -40,7 +40,7 @@ module Gitlab ...@@ -40,7 +40,7 @@ module Gitlab
if strategy.enabled? if strategy.enabled?
models_with_missing_counts = models - counts_by_model.keys models_with_missing_counts = models - counts_by_model.keys
break if models_with_missing_counts.empty? break counts_by_model if models_with_missing_counts.empty?
counts = strategy.new(models_with_missing_counts).count counts = strategy.new(models_with_missing_counts).count
......
...@@ -20,6 +20,8 @@ module Gitlab ...@@ -20,6 +20,8 @@ module Gitlab
models.each_with_object({}) do |model, data| models.each_with_object({}) do |model, data|
data[model] = model.count data[model] = model.count
end end
rescue *CONNECTION_ERRORS
{}
end end
def self.enabled? def self.enabled?
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
module Gitlab module Gitlab
class UsageData class UsageData
APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze
class << self class << self
def data(force_refresh: false) def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
...@@ -73,12 +75,9 @@ module Gitlab ...@@ -73,12 +75,9 @@ module Gitlab
issues: count(Issue), issues: count(Issue),
keys: count(Key), keys: count(Key),
label_lists: count(List.label), label_lists: count(List.label),
labels: count(Label),
lfs_objects: count(LfsObject), lfs_objects: count(LfsObject),
merge_requests: count(MergeRequest),
milestone_lists: count(List.milestone), milestone_lists: count(List.milestone),
milestones: count(Milestone), milestones: count(Milestone),
notes: count(Note),
pages_domains: count(PagesDomain), pages_domains: count(PagesDomain),
projects: count(Project), projects: count(Project),
projects_imported_from_github: count(Project.where(import_type: 'github')), projects_imported_from_github: count(Project.where(import_type: 'github')),
...@@ -86,10 +85,9 @@ module Gitlab ...@@ -86,10 +85,9 @@ module Gitlab
releases: count(Release), releases: count(Release),
remote_mirrors: count(RemoteMirror), remote_mirrors: count(RemoteMirror),
snippets: count(Snippet), snippets: count(Snippet),
todos: count(Todo),
uploads: count(Upload), uploads: count(Upload),
web_hooks: count(WebHook) web_hooks: count(WebHook)
}.merge(services_usage) }.merge(services_usage).merge(approximate_counts)
} }
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
...@@ -164,6 +162,16 @@ module Gitlab ...@@ -164,6 +162,16 @@ module Gitlab
fallback fallback
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def approximate_counts
approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS)
APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result|
key = model.name.underscore.pluralize.to_sym
result[key] = approx_counts[model] || -1
end
end
end end
end end
end end
...@@ -16,6 +16,12 @@ describe Gitlab::Database::Count::ExactCountStrategy do ...@@ -16,6 +16,12 @@ describe Gitlab::Database::Count::ExactCountStrategy do
expect(subject).to eq({ Project => 3, Identity => 1 }) expect(subject).to eq({ Project => 3, Identity => 1 })
end end
it 'returns default value if count times out' do
allow(models.first).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(subject).to eq({})
end
end end
describe '.enabled?' do describe '.enabled?' do
......
...@@ -213,4 +213,29 @@ describe Gitlab::UsageData do ...@@ -213,4 +213,29 @@ describe Gitlab::UsageData do
expect(described_class.count(relation, fallback: 15)).to eq(15) expect(described_class.count(relation, fallback: 15)).to eq(15)
end end
end end
describe '#approximate_counts' do
it 'gets approximate counts for selected models' do
create(:label)
expect(Gitlab::Database::Count).to receive(:approximate_counts)
.with(described_class::APPROXIMATE_COUNT_MODELS).once.and_call_original
counts = described_class.approximate_counts.values
expect(counts.count).to eq(described_class::APPROXIMATE_COUNT_MODELS.count)
expect(counts.any? { |count| count < 0 }).to be_falsey
end
it 'returns default values if counts can not be retrieved' do
described_class::APPROXIMATE_COUNT_MODELS.map do |model|
model.name.underscore.pluralize.to_sym
end
expect(Gitlab::Database::Count).to receive(:approximate_counts)
.and_return({})
expect(described_class.approximate_counts.values.uniq).to eq([-1])
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment