Commit a82aabe4 authored by Ash McKenzie's avatar Ash McKenzie

Merge branch 'add_measurement_to_import_rake_task' into 'master'

Add measurement to import_rake task

See merge request gitlab-org/gitlab!24475
parents 93539739 3d94dd84
......@@ -53,8 +53,18 @@ As part of this script we also disable direct and background upload to avoid sit
We can simply run this script from the terminal:
Parameters:
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `username` | string | yes | User name |
| `namespace_path` | string | yes | Namespace path |
| `project_path` | string | yes | Project name |
| `archive_path` | string | yes | Path to the exported project tarball you want to import |
| `measurement_enabled` | boolean | no | Measure execution time, number of SQL calls and GC count |
```shell
bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz]"
bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz, true]"
```
### Importing via the Rails console
......
......@@ -7,12 +7,12 @@
# 2. Performs Sidekiq job synchronously
#
# @example
# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]"
# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]"
#
namespace :gitlab do
namespace :import_export do
desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
# Load it here to avoid polluting Rake tasks with Sidekiq test warnings
require 'sidekiq/testing'
......@@ -26,7 +26,8 @@ namespace :gitlab do
namespace_path: args.namespace_path,
project_path: args.project_path,
username: args.username,
file_path: args.archive_path
file_path: args.archive_path,
measurement_enabled: args.measurement_enabled == 'true'
).import
end
end
......@@ -38,6 +39,7 @@ class GitlabProjectImport
@file_path = opts.fetch(:file_path)
@namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path))
@current_user = User.find_by_username(opts.fetch(:username))
@measurement_enabled = opts.fetch(:measurement_enabled)
end
def import
......@@ -72,6 +74,54 @@ class GitlabProjectImport
RequestStore.clear!
end
def with_count_queries(&block)
count = 0
counter_f = ->(name, started, finished, unique_id, payload) {
unless payload[:name].in? %w[CACHE SCHEMA]
count += 1
end
}
ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
puts "Number of sql calls: #{count}"
end
def with_gc_counter
gc_counts_before = GC.stat.select { |k, v| k =~ /count/ }
yield
gc_counts_after = GC.stat.select { |k, v| k =~ /count/ }
stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb }
puts "Total GC count: #{stats[:count]}"
puts "Minor GC count: #{stats[:minor_gc_count]}"
puts "Major GC count: #{stats[:major_gc_count]}"
end
def with_measure_time
timing = Benchmark.realtime do
yield
end
time = Time.at(timing).utc.strftime("%H:%M:%S")
puts "Time to finish: #{time}"
end
def with_measuring
puts "Measuring enabled..."
with_gc_counter do
with_count_queries do
with_measure_time do
yield
end
end
end
end
def measurement_enabled?
@measurement_enabled != false
end
# We want to ensure that all Sidekiq jobs are executed
# synchronously as part of that process.
# This ensures that all expensive operations do not escape
......@@ -79,8 +129,13 @@ class GitlabProjectImport
def with_isolated_sidekiq_job
Sidekiq::Testing.fake! do
with_request_store do
# If you are attempting to import a large project into a development environment,
# you may see Gitaly throw an error about too many calls or invocations.
# This is due to a n+1 calls limit being set for development setups (not enforced in production)
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
# For development setups, this code-path will be excluded from n+1 detection.
::Gitlab::GitalyClient.allow_n_plus_1_calls do
yield
measurement_enabled? ? with_measuring { yield } : yield
end
end
......
# frozen_string_literal: true
RSpec.shared_examples 'import measurement' do
context 'when measurement is enabled' do
let(:measurement_enabled) { true }
it 'prints measurement results' do
expect { subject }.to output(including('Measuring enabled...', 'Number of sql calls:', 'Total GC count:', 'Total GC count:')).to_stdout
end
end
context 'when measurement is not enabled' do
let(:measurement_enabled) { false }
it 'does not output measurement results' do
expect { subject }.not_to output(/Measuring enabled.../).to_stdout
end
end
context 'when measurement is not provided' do
let(:task_params) { [username, namespace_path, project_name, archive_path] }
it 'does not output measurement results' do
expect { subject }.not_to output(/Measuring enabled.../).to_stdout
end
it 'does not raise any exception' do
expect { subject }.not_to raise_error
end
end
end
......@@ -6,7 +6,8 @@ describe 'gitlab:import_export:import rake task' do
let(:username) { 'root' }
let(:namespace_path) { username }
let!(:user) { create(:user, username: username) }
let(:task_params) { [username, namespace_path, project_name, archive_path] }
let(:measurement_enabled) { false }
let(:task_params) { [username, namespace_path, project_name, archive_path, measurement_enabled] }
let(:project) { Project.find_by_full_path("#{namespace_path}/#{project_name}") }
before do
......@@ -68,6 +69,8 @@ describe 'gitlab:import_export:import rake task' do
subject
end
it_behaves_like 'import measurement'
end
context 'when project import is invalid' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment