Commit e0934f66 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'ee-delay-update-statictics' into 'master'

EE port of fixing the bug that the project statistics is not updated

See merge request gitlab-org/gitlab-ee!10626
parents fd92ae30 e0451079
# frozen_string_literal: true
module Projects
class UpdateStatisticsService < BaseService
def execute
return unless project && project.repository.exists?
Rails.logger.info("Updating statistics for project #{project.id}")
project.statistics.refresh!(only: statistics.map(&:to_sym))
end
private
def statistics
params[:statistics]
end
end
end
......@@ -143,6 +143,7 @@
- repository_remove_remote
- system_hook_push
- update_merge_requests
- update_project_statistics
- upload_checksum
- web_hook
- repository_update_remote_mirror
......
......@@ -17,7 +17,7 @@ class ProjectCacheWorker
return unless project && project.repository.exists?
update_statistics(project, statistics.map(&:to_sym))
update_statistics(project, statistics)
project.repository.refresh_method_caches(files.map(&:to_sym))
......@@ -25,22 +25,30 @@ class ProjectCacheWorker
end
# rubocop: enable CodeReuse/ActiveRecord
# NOTE: triggering both an immediate update and one in 15 minutes if we
# successfully obtain the lease. That way, we only need to wait for the
# statistics to become accurate if they were already updated once in the
# last 15 minutes.
def update_statistics(project, statistics = [])
return if Gitlab::Database.read_only?
return unless try_obtain_lease_for(project.id, :update_statistics)
return unless try_obtain_lease_for(project.id, statistics)
Rails.logger.info("Updating statistics for project #{project.id}")
Projects::UpdateStatisticsService.new(project, nil, statistics: statistics).execute
project.statistics.refresh!(only: statistics)
UpdateProjectStatisticsWorker.perform_in(LEASE_TIMEOUT, project.id, statistics)
end
private
def try_obtain_lease_for(project_id, section)
def try_obtain_lease_for(project_id, statistics)
Gitlab::ExclusiveLease
.new("project_cache_worker:#{project_id}:#{section}", timeout: LEASE_TIMEOUT)
.new(project_cache_worker_key(project_id, statistics), timeout: LEASE_TIMEOUT)
.try_obtain
end
def project_cache_worker_key(project_id, statistics)
["project_cache_worker", project_id, *statistics.sort].join(":")
end
end
ProjectCacheWorker.prepend(EE::Workers::ProjectCacheWorker)
# frozen_string_literal: true
# Worker for updating project statistics.
class UpdateProjectStatisticsWorker
include ApplicationWorker
# project_id - The ID of the project for which to flush the cache.
# statistics - An Array containing columns from ProjectStatistics to
# refresh, if empty all columns will be refreshed
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, statistics = [])
project = Project.find_by(id: project_id)
Projects::UpdateStatisticsService.new(project, nil, statistics: statistics).execute
end
# rubocop: enable CodeReuse/ActiveRecord
end
---
title: Fix the bug that the project statistics is not updated
merge_request: 26854
author: Hiroyuki Sato
type: fixed
......@@ -89,7 +89,8 @@
- [project_daily_statistics, 1]
- [import_issues_csv, 2]
- [chat_notification, 2]
- [migrate_external_diffs, 1]
- [migrate_external_diffs, 1]
- [update_project_statistics, 1]
# EE-specific queues
- [ldap_group_sync, 2]
......
require 'spec_helper'
describe Projects::UpdateStatisticsService do
let(:service) { described_class.new(project, nil, statistics: statistics)}
let(:statistics) { %w(repository_size) }
describe '#execute' do
context 'with a non-existing project' do
let(:project) { nil }
it 'does nothing' do
expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!)
service.execute
end
end
context 'with an existing project without a repository' do
let(:project) { create(:project) }
it 'does nothing' do
expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!)
service.execute
end
end
context 'with an existing project with a repository' do
let(:project) { create(:project, :repository) }
it 'refreshes the project statistics' do
expect_any_instance_of(ProjectStatistics).to receive(:refresh!)
.with(only: statistics.map(&:to_sym))
.and_call_original
service.execute
end
end
end
end
......@@ -7,9 +7,9 @@ describe ProjectCacheWorker do
let(:worker) { described_class.new }
let(:project) { create(:project, :repository) }
let(:statistics) { project.statistics }
let(:lease_key) { "project_cache_worker:#{project.id}:update_statistics" }
let(:lease_key) { ["project_cache_worker", project.id, *statistics.sort].join(":") }
let(:lease_timeout) { ProjectCacheWorker::LEASE_TIMEOUT }
let(:statistics) { [] }
describe '#perform' do
before do
......@@ -35,14 +35,6 @@ describe ProjectCacheWorker do
end
context 'with an existing project' do
it 'updates the project statistics' do
expect(worker).to receive(:update_statistics)
.with(kind_of(Project), %i(repository_size))
.and_call_original
worker.perform(project.id, [], %w(repository_size))
end
it 'refreshes the method caches' do
expect_any_instance_of(Repository).to receive(:refresh_method_caches)
.with(%i(readme))
......@@ -67,6 +59,18 @@ describe ProjectCacheWorker do
end
end
context 'with statistics' do
let(:statistics) { %w(repository_size) }
it 'updates the project statistics' do
expect(worker).to receive(:update_statistics)
.with(kind_of(Project), statistics)
.and_call_original
worker.perform(project.id, [], statistics)
end
end
context 'with plain readme' do
it 'refreshes the method caches' do
allow(MarkupHelper).to receive(:gitlab_markdown?).and_return(false)
......@@ -82,25 +86,34 @@ describe ProjectCacheWorker do
end
describe '#update_statistics' do
let(:statistics) { %w(repository_size) }
context 'when a lease could not be obtained' do
it 'does not update the repository size' do
it 'does not update the project statistics' do
stub_exclusive_lease_taken(lease_key, timeout: lease_timeout)
expect(statistics).not_to receive(:refresh!)
expect(Projects::UpdateStatisticsService).not_to receive(:new)
worker.update_statistics(project)
expect(UpdateProjectStatisticsWorker).not_to receive(:perform_in)
worker.update_statistics(project, statistics)
end
end
context 'when a lease could be obtained' do
it 'updates the project statistics' do
it 'updates the project statistics twice' do
stub_exclusive_lease(lease_key, timeout: lease_timeout)
expect(statistics).to receive(:refresh!)
.with(only: %i(repository_size))
expect(Projects::UpdateStatisticsService).to receive(:new)
.with(project, nil, statistics: statistics)
.and_call_original
.twice
expect(UpdateProjectStatisticsWorker).to receive(:perform_in)
.with(lease_timeout, project.id, statistics)
.and_call_original
worker.update_statistics(project, %i(repository_size))
worker.update_statistics(project, statistics)
end
end
end
......
require 'spec_helper'
describe UpdateProjectStatisticsWorker do
let(:worker) { described_class.new }
let(:project) { create(:project, :repository) }
let(:statistics) { %w(repository_size) }
describe '#perform' do
it 'updates the project statistics' do
expect(Projects::UpdateStatisticsService).to receive(:new)
.with(project, nil, statistics: statistics)
.and_call_original
worker.perform(project.id, statistics)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment