Populate personal snippet statistics

This migration updates/create personal snippet
statiscs and update the related namespace ones.
parent d779c8b1
---
title: Backfill personal snippets statistics
merge_request: 36801
author:
type: other
# frozen_string_literal: true
class SchedulePopulatePersonalSnippetStatistics < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
DELAY_INTERVAL = 2.minutes.to_i
BATCH_SIZE = 500
MIGRATION = 'PopulatePersonalSnippetStatistics'
disable_ddl_transaction!
def up
snippets = exec_query <<~SQL
SELECT id
FROM snippets
WHERE type = 'PersonalSnippet'
ORDER BY author_id ASC, id ASC
SQL
snippets.rows.flatten.in_groups_of(BATCH_SIZE, false).each_with_index do |snippet_ids, index|
migrate_in(index * DELAY_INTERVAL, MIGRATION, [snippet_ids])
end
end
def down
# no-op
end
end
......@@ -23990,6 +23990,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200713071042
20200713141854
20200713152443
20200714075739
20200715124210
20200715135130
20200715202659
......
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This class creates/updates those personal snippets statistics
# that haven't been created nor initialized.
# It also updates the related root storage namespace stats
class PopulatePersonalSnippetStatistics
def perform(snippet_ids)
personal_snippets(snippet_ids).group_by(&:author).each do |author, author_snippets|
upsert_snippet_statistics(author_snippets)
update_namespace_statistics(author.namespace)
end
end
private
def personal_snippets(snippet_ids)
PersonalSnippet
.where(id: snippet_ids)
.includes(author: :namespace)
.includes(:statistics)
.includes(snippet_repository: :shard)
end
def upsert_snippet_statistics(snippets)
snippets.each do |snippet|
response = Snippets::UpdateStatisticsService.new(snippet).execute
error_message("#{response.message} snippet: #{snippet.id}") if response.error?
end
end
def update_namespace_statistics(namespace)
Namespaces::StatisticsRefresherService.new.execute(namespace)
rescue => e
error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}")
end
def logger
@logger ||= Gitlab::BackgroundMigration::Logger.build
end
def error_message(message)
logger.error(message: "Snippet Statistics Migration: #{message}")
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::PopulatePersonalSnippetStatistics do
let(:file_name) { 'file_name.rb' }
let(:content) { 'content' }
let(:snippets) { table(:snippets) }
let(:snippet_repositories) { table(:snippet_repositories) }
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:snippet_statistics) { table(:snippet_statistics) }
let(:namespace_statistics) { table(:namespace_root_storage_statistics) }
let(:routes) { table(:routes) }
let(:repo_size) { 123456 }
let(:expected_repo_size) { repo_size.megabytes }
let(:user1) { users.create!(id: 1, email: 'test@example.com', projects_limit: 100, username: 'test1') }
let(:user2) { users.create!(id: 2, email: 'test2@example.com', projects_limit: 100, username: 'test2') }
let!(:user1_namespace) { namespaces.create!(id: 1, name: 'user1', path: 'user1', owner_id: user1.id) }
let!(:user2_namespace) { namespaces.create!(id: 2, name: 'user2', path: 'user2', owner_id: user2.id) }
let(:user1_namespace_statistics) { namespace_statistics.find_by(namespace_id: user1_namespace.id) }
let(:user2_namespace_statistics) { namespace_statistics.find_by(namespace_id: user2_namespace.id) }
let(:ids) { snippets.pluck(:id) }
let(:migration) { described_class.new }
subject do
migration.perform(ids)
end
before do
allow_any_instance_of(Repository).to receive(:size).and_return(repo_size)
end
after do
snippets.all.each { |s| raw_repository(s).remove }
end
context 'with existing personal snippets' do
let!(:snippet1) { create_snippet(1, user1) }
let!(:snippet2) { create_snippet(2, user1) }
let!(:snippet3) { create_snippet(3, user2) }
let!(:snippet4) { create_snippet(4, user2) }
before do
create_snippet_statistics(2, 0)
create_snippet_statistics(4, 123)
end
it 'creates/updates all snippet_statistics' do
expect { subject }.to change { snippet_statistics.count }.from(2).to(4)
expect(snippet_statistics.pluck(:repository_size)).to be_all(expected_repo_size)
end
it 'creates/updates the associated namespace statistics' do
expect(migration).to receive(:update_namespace_statistics).twice.and_call_original
subject
stats = snippet_statistics.where(snippet_id: [snippet1, snippet2]).sum(:repository_size)
expect(user1_namespace_statistics.snippets_size).to eq stats
stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size)
expect(user2_namespace_statistics.snippets_size).to eq stats
end
context 'when an error is raised when updating a namespace statistics' do
it 'logs the error and continue execution' do
expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance|
expect(instance).to receive(:execute).with(Namespace.find(user1_namespace.id)).and_raise('Error')
end
expect_next_instance_of(Namespaces::StatisticsRefresherService) do |instance|
expect(instance).to receive(:execute).and_call_original
end
expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance|
expect(instance).to receive(:error).with(message: /Error updating statistics for namespace/).once
end
subject
expect(user1_namespace_statistics).to be_nil
stats = snippet_statistics.where(snippet_id: [snippet3, snippet4]).sum(:repository_size)
expect(user2_namespace_statistics.snippets_size).to eq stats
end
end
end
context 'when a snippet repository is empty' do
let!(:snippet1) { create_snippet(1, user1, with_repo: false) }
let!(:snippet2) { create_snippet(2, user1) }
it 'logs error and continues execution' do
expect_next_instance_of(Gitlab::BackgroundMigration::Logger) do |instance|
expect(instance).to receive(:error).with(message: /Invalid snippet repository/).once
end
subject
expect(snippet_statistics.find_by(snippet_id: snippet1.id)).to be_nil
expect(user1_namespace_statistics.snippets_size).to eq expected_repo_size
end
end
def create_snippet(id, author, with_repo: true)
snippets.create!(id: id, type: 'PersonalSnippet', author_id: author.id, file_name: file_name, content: content).tap do |snippet|
if with_repo
allow(snippet).to receive(:disk_path).and_return(disk_path(snippet))
TestEnv.copy_repo(snippet,
bare_repo: TestEnv.factory_repo_path_bare,
refs: TestEnv::BRANCH_SHA)
raw_repository(snippet).create_repository
end
end
end
def create_snippet_statistics(snippet_id, repository_size = 0)
snippet_statistics.create!(snippet_id: snippet_id, repository_size: repository_size)
end
def raw_repository(snippet)
Gitlab::Git::Repository.new('default',
"#{disk_path(snippet)}.git",
Gitlab::GlRepository::SNIPPET.identifier_for_container(snippet),
"@snippets/#{snippet.id}")
end
def hashed_repository(snippet)
Storage::Hashed.new(snippet, prefix: '@snippets')
end
def disk_path(snippet)
hashed_repository(snippet).disk_path
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200714075739_schedule_populate_personal_snippet_statistics.rb')
RSpec.describe SchedulePopulatePersonalSnippetStatistics do
let(:users) { table(:users) }
let(:snippets) { table(:snippets) }
let(:projects) { table(:projects) }
let(:user1) { users.create!(id: 1, email: 'user1@example.com', projects_limit: 10, username: 'test1', name: 'Test1', state: 'active') }
let(:user2) { users.create!(id: 2, email: 'user2@example.com', projects_limit: 10, username: 'test2', name: 'Test2', state: 'active') }
let(:user3) { users.create!(id: 3, email: 'user3@example.com', projects_limit: 10, username: 'test3', name: 'Test3', state: 'active') }
def create_snippet(id, user_id, type = 'PersonalSnippet')
params = {
id: id,
type: type,
author_id: user_id,
file_name: 'foo',
content: 'bar'
}
snippets.create!(params)
end
it 'correctly schedules background migrations' do
# Creating the snippets in different order
create_snippet(1, user1.id)
create_snippet(2, user2.id)
create_snippet(3, user1.id)
create_snippet(4, user3.id)
create_snippet(5, user3.id)
create_snippet(6, user1.id)
# Creating a project snippet to ensure we don't pick it
create_snippet(7, user1.id, 'ProjectSnippet')
stub_const("#{described_class}::BATCH_SIZE", 4)
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
aggregate_failures do
expect(described_class::MIGRATION)
.to be_scheduled_migration([1, 3, 6, 2])
expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(2.minutes, [4, 5])
expect(BackgroundMigrationWorker.jobs.size).to eq(2)
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment