Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
492d7096
Commit
492d7096
authored
Feb 25, 2020
by
Nikola Milojevic
Committed by
Peter Leitzen
Feb 25, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add export task
- Add measurable - Add copy strategy
parent
feef9ebc
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
282 additions
and
90 deletions
+282
-90
lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
...port_export/after_export_strategies/move_file_strategy.rb
+19
-0
lib/gitlab/profiler.rb
lib/gitlab/profiler.rb
+16
-16
lib/gitlab/sidekiq_middleware/request_store_middleware.rb
lib/gitlab/sidekiq_middleware/request_store_middleware.rb
+5
-5
lib/gitlab/utils/measuring.rb
lib/gitlab/utils/measuring.rb
+74
-0
lib/gitlab/with_request_store.rb
lib/gitlab/with_request_store.rb
+13
-0
lib/tasks/gitlab/import_export/export.rake
lib/tasks/gitlab/import_export/export.rake
+94
-0
lib/tasks/gitlab/import_export/import.rake
lib/tasks/gitlab/import_export/import.rake
+22
-67
spec/support/shared_examples/tasks/gitlab/import_export/measurable_shared_examples.rb
.../tasks/gitlab/import_export/measurable_shared_examples.rb
+1
-1
spec/tasks/gitlab/import_export/export_rake_spec.rb
spec/tasks/gitlab/import_export/export_rake_spec.rb
+37
-0
spec/tasks/gitlab/import_export/import_rake_spec.rb
spec/tasks/gitlab/import_export/import_rake_spec.rb
+1
-1
No files found.
lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
0 → 100644
View file @
492d7096
# frozen_string_literal: true
module
Gitlab
module
ImportExport
module
AfterExportStrategies
class
MoveFileStrategy
<
BaseAfterExportStrategy
def
initialize
(
archive_path
:)
@archive_path
=
archive_path
end
private
def
strategy_execute
FileUtils
.
mv
(
project
.
export_file
.
path
,
@archive_path
)
end
end
end
end
end
lib/gitlab/profiler.rb
View file @
492d7096
...
...
@@ -2,6 +2,8 @@
module
Gitlab
module
Profiler
extend
WithRequestStore
FILTERED_STRING
=
'[FILTERED]'
IGNORE_BACKTRACES
=
%w[
...
...
@@ -58,28 +60,26 @@ module Gitlab
logger
=
create_custom_logger
(
logger
,
private_token:
private_token
)
RequestStore
.
begin!
# Make an initial call for an asset path in development mode to avoid
# sprockets dominating the profiler output.
ActionController
::
Base
.
helpers
.
asset_path
(
'katex.css'
)
if
Rails
.
env
.
development?
result
=
with_request_store
do
# Make an initial call for an asset path in development mode to avoid
# sprockets dominating the profiler output.
ActionController
::
Base
.
helpers
.
asset_path
(
'katex.css'
)
if
Rails
.
env
.
development?
# Rails loads internationalization files lazily the first time a
# translation is needed. Running this prevents this overhead from showing
# up in profiles.
::
I18n
.
t
(
'.'
)[
:test_string
]
# Rails loads internationalization files lazily the first time a
# translation is needed. Running this prevents this overhead from showing
# up in profiles.
::
I18n
.
t
(
'.'
)[
:test_string
]
# Remove API route mounting from the profile.
app
.
get
(
'/api/v4/users'
)
# Remove API route mounting from the profile.
app
.
get
(
'/api/v4/users'
)
result
=
with_custom_logger
(
logger
)
do
with_user
(
user
)
do
RubyProf
.
profile
{
app
.
public_send
(
verb
,
url
,
params:
post_data
,
headers:
headers
)
}
# rubocop:disable GitlabSecurity/PublicSend
with_custom_logger
(
logger
)
do
with_user
(
user
)
do
RubyProf
.
profile
{
app
.
public_send
(
verb
,
url
,
params:
post_data
,
headers:
headers
)
}
# rubocop:disable GitlabSecurity/PublicSend
end
end
end
RequestStore
.
end!
log_load_times_by_model
(
logger
)
result
...
...
lib/gitlab/sidekiq_middleware/request_store_middleware.rb
View file @
492d7096
...
...
@@ -3,12 +3,12 @@
module
Gitlab
module
SidekiqMiddleware
class
RequestStoreMiddleware
include
Gitlab
::
WithRequestStore
def
call
(
worker
,
job
,
queue
)
RequestStore
.
begin!
yield
ensure
RequestStore
.
end!
RequestStore
.
clear!
with_request_store
do
yield
end
end
end
end
...
...
lib/gitlab/utils/measuring.rb
0 → 100644
View file @
492d7096
# frozen_string_literal: true
require
'prometheus/pid_provider'
module
Gitlab
module
Utils
class
Measuring
def
initialize
(
logger:
Logger
.
new
(
$stdout
))
@logger
=
logger
end
def
with_measuring
logger
.
info
"Measuring enabled..."
with_gc_counter
do
with_count_queries
do
with_measure_time
do
yield
end
end
end
logger
.
info
"Memory usage:
#{
Gitlab
::
Metrics
::
System
.
memory_usage
.
to_f
/
1024
/
1024
}
MiB"
logger
.
info
"Label:
#{
::
Prometheus
::
PidProvider
.
worker_id
}
"
end
private
attr_reader
:logger
def
with_count_queries
(
&
block
)
count
=
0
counter_f
=
->
(
_name
,
_started
,
_finished
,
_unique_id
,
payload
)
{
count
+=
1
unless
payload
[
:name
].
in?
%w[CACHE SCHEMA]
}
ActiveSupport
::
Notifications
.
subscribed
(
counter_f
,
"sql.active_record"
,
&
block
)
logger
.
info
"Number of sql calls:
#{
count
}
"
end
def
with_gc_counter
gc_counts_before
=
GC
.
stat
.
select
{
|
k
,
_v
|
k
=~
/count/
}
yield
gc_counts_after
=
GC
.
stat
.
select
{
|
k
,
_v
|
k
=~
/count/
}
stats
=
gc_counts_before
.
merge
(
gc_counts_after
)
{
|
_k
,
vb
,
va
|
va
-
vb
}
logger
.
info
"Total GC count:
#{
stats
[
:count
]
}
"
logger
.
info
"Minor GC count:
#{
stats
[
:minor_gc_count
]
}
"
logger
.
info
"Major GC count:
#{
stats
[
:major_gc_count
]
}
"
end
def
with_measure_time
timing
=
Benchmark
.
realtime
do
yield
end
logger
.
info
"Time to finish:
#{
duration_in_numbers
(
timing
)
}
"
end
def
duration_in_numbers
(
duration_in_seconds
)
seconds
=
duration_in_seconds
%
1
.
minute
minutes
=
(
duration_in_seconds
/
1
.
minute
)
%
(
1
.
hour
/
1
.
minute
)
hours
=
duration_in_seconds
/
1
.
hour
if
hours
==
0
"%02d:%02d"
%
[
minutes
,
seconds
]
else
"%02d:%02d:%02d"
%
[
hours
,
minutes
,
seconds
]
end
end
end
end
end
lib/gitlab/with_request_store.rb
0 → 100644
View file @
492d7096
# frozen_string_literal: true
module
Gitlab
module
WithRequestStore
def
with_request_store
RequestStore
.
begin!
yield
ensure
RequestStore
.
end!
RequestStore
.
clear!
end
end
end
lib/tasks/gitlab/import_export/export.rake
0 → 100644
View file @
492d7096
# frozen_string_literal: true
require
'gitlab/with_request_store'
# Export project to archive
#
# @example
# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]"
#
namespace
:gitlab
do
namespace
:import_export
do
desc
'GitLab | Import/Export | EXPERIMENTAL | Export large project archives'
task
:export
,
[
:username
,
:namespace_path
,
:project_path
,
:archive_path
,
:measurement_enabled
]
=>
:gitlab_environment
do
|
_t
,
args
|
# Load it here to avoid polluting Rake tasks with Sidekiq test warnings
require
'sidekiq/testing'
warn_user_is_not_gitlab
if
ENV
[
'IMPORT_DEBUG'
].
present?
ActiveRecord
::
Base
.
logger
=
Logger
.
new
(
STDOUT
)
Gitlab
::
Metrics
::
Exporter
::
SidekiqExporter
.
instance
.
start
end
GitlabProjectExport
.
new
(
namespace_path:
args
.
namespace_path
,
project_path:
args
.
project_path
,
username:
args
.
username
,
file_path:
args
.
archive_path
,
measurement_enabled:
Gitlab
::
Utils
.
to_boolean
(
args
.
measurement_enabled
)
).
export
end
end
end
class
GitlabProjectExport
include
Gitlab
::
WithRequestStore
def
initialize
(
opts
)
@project_path
=
opts
.
fetch
(
:project_path
)
@file_path
=
opts
.
fetch
(
:file_path
)
@current_user
=
User
.
find_by_username
(
opts
.
fetch
(
:username
))
namespace
=
Namespace
.
find_by_full_path
(
opts
.
fetch
(
:namespace_path
))
@project
=
namespace
.
projects
.
find_by_path
(
@project_path
)
@measurement_enabled
=
opts
.
fetch
(
:measurement_enabled
)
@measurable
=
Gitlab
::
Utils
::
Measuring
.
new
if
@measurement_enabled
end
def
export
validate_project
validate_file_path
with_export
do
::
Projects
::
ImportExport
::
ExportService
.
new
(
project
,
current_user
)
.
execute
(
Gitlab
::
ImportExport
::
AfterExportStrategies
::
MoveFileStrategy
.
new
(
archive_path:
file_path
))
end
puts
'Done!'
rescue
StandardError
=>
e
puts
"Exception:
#{
e
.
message
}
"
puts
e
.
backtrace
exit
1
end
private
attr_reader
:measurable
,
:project
,
:current_user
,
:file_path
,
:project_path
def
validate_project
unless
project
puts
"Error: Project with path:
#{
project_path
}
was not found. Please provide correct project path"
exit
1
end
end
def
validate_file_path
directory
=
File
.
dirname
(
file_path
)
unless
Dir
.
exist?
(
directory
)
puts
"Error: Invalid file path:
#{
file_path
}
. Please provide correct file path"
exit
1
end
end
def
with_export
with_request_store
do
::
Gitlab
::
GitalyClient
.
allow_n_plus_1_calls
do
measurement_enabled?
?
measurable
.
with_measuring
{
yield
}
:
yield
end
end
end
def
measurement_enabled?
@measurement_enabled
end
end
lib/tasks/gitlab/import_export/import.rake
View file @
492d7096
# frozen_string_literal: true
require
'gitlab/with_request_store'
# Import large project archives
#
# This task:
...
...
@@ -27,19 +29,22 @@ namespace :gitlab do
project_path:
args
.
project_path
,
username:
args
.
username
,
file_path:
args
.
archive_path
,
measurement_enabled:
args
.
measurement_enabled
==
'true'
measurement_enabled:
Gitlab
::
Utils
.
to_boolean
(
args
.
measurement_enabled
)
).
import
end
end
end
class
GitlabProjectImport
include
Gitlab
::
WithRequestStore
def
initialize
(
opts
)
@project_path
=
opts
.
fetch
(
:project_path
)
@file_path
=
opts
.
fetch
(
:file_path
)
@namespace
=
Namespace
.
find_by_full_path
(
opts
.
fetch
(
:namespace_path
))
@current_user
=
User
.
find_by_username
(
opts
.
fetch
(
:username
))
@measurement_enabled
=
opts
.
fetch
(
:measurement_enabled
)
@measurement
=
Gitlab
::
Utils
::
Measuring
.
new
if
@measurement_enabled
end
def
import
...
...
@@ -49,11 +54,11 @@ class GitlabProjectImport
show_import_failures_count
if
@
project
&
.
import_state
&
.
last_error
puts
"ERROR:
#{
@
project
.
import_state
.
last_error
}
"
if
project
&
.
import_state
&
.
last_error
puts
"ERROR:
#{
project
.
import_state
.
last_error
}
"
exit
1
elsif
@
project
.
errors
.
any?
puts
"ERROR:
#{
@
project
.
errors
.
full_messages
.
join
(
', '
)
}
"
elsif
project
.
errors
.
any?
puts
"ERROR:
#{
project
.
errors
.
full_messages
.
join
(
', '
)
}
"
exit
1
else
puts
'Done!'
...
...
@@ -66,60 +71,10 @@ class GitlabProjectImport
private
def
with_request_store
RequestStore
.
begin!
yield
ensure
RequestStore
.
end!
RequestStore
.
clear!
end
def
with_count_queries
(
&
block
)
count
=
0
counter_f
=
->
(
name
,
started
,
finished
,
unique_id
,
payload
)
{
unless
payload
[
:name
].
in?
%w[CACHE SCHEMA]
count
+=
1
end
}
ActiveSupport
::
Notifications
.
subscribed
(
counter_f
,
"sql.active_record"
,
&
block
)
puts
"Number of sql calls:
#{
count
}
"
end
def
with_gc_counter
gc_counts_before
=
GC
.
stat
.
select
{
|
k
,
v
|
k
=~
/count/
}
yield
gc_counts_after
=
GC
.
stat
.
select
{
|
k
,
v
|
k
=~
/count/
}
stats
=
gc_counts_before
.
merge
(
gc_counts_after
)
{
|
k
,
vb
,
va
|
va
-
vb
}
puts
"Total GC count:
#{
stats
[
:count
]
}
"
puts
"Minor GC count:
#{
stats
[
:minor_gc_count
]
}
"
puts
"Major GC count:
#{
stats
[
:major_gc_count
]
}
"
end
def
with_measure_time
timing
=
Benchmark
.
realtime
do
yield
end
time
=
Time
.
at
(
timing
).
utc
.
strftime
(
"%H:%M:%S"
)
puts
"Time to finish:
#{
time
}
"
end
def
with_measuring
puts
"Measuring enabled..."
with_gc_counter
do
with_count_queries
do
with_measure_time
do
yield
end
end
end
end
attr_reader
:measurement
,
:project
,
:namespace
,
:current_user
,
:file_path
,
:project_path
def
measurement_enabled?
@measurement_enabled
!=
false
@measurement_enabled
end
# We want to ensure that all Sidekiq jobs are executed
...
...
@@ -135,7 +90,7 @@ class GitlabProjectImport
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
# For development setups, this code-path will be excluded from n+1 detection.
::
Gitlab
::
GitalyClient
.
allow_n_plus_1_calls
do
measurement_enabled?
?
with_measuring
{
yield
}
:
yield
measurement_enabled?
?
measurement
.
with_measuring
{
yield
}
:
yield
end
end
...
...
@@ -158,11 +113,11 @@ class GitlabProjectImport
# 2. Download of archive before unpacking
disable_upload_object_storage
do
service
=
Projects
::
GitlabProjectsImportService
.
new
(
@
current_user
,
current_user
,
{
namespace_id:
@
namespace
.
id
,
path:
@
project_path
,
file:
File
.
open
(
@
file_path
)
namespace_id:
namespace
.
id
,
path:
project_path
,
file:
File
.
open
(
file_path
)
}
)
...
...
@@ -193,18 +148,18 @@ class GitlabProjectImport
end
def
full_path
"
#{
@namespace
.
full_path
}
/
#{
@
project_path
}
"
"
#{
namespace
.
full_path
}
/
#{
project_path
}
"
end
def
show_import_start_message
puts
"Importing GitLab export:
#{
@
file_path
}
into GitLab"
\
puts
"Importing GitLab export:
#{
file_path
}
into GitLab"
\
"
#{
full_path
}
"
\
" as
#{
@
current_user
.
name
}
"
" as
#{
current_user
.
name
}
"
end
def
show_import_failures_count
return
unless
@
project
.
import_failures
.
exists?
return
unless
project
.
import_failures
.
exists?
puts
"Total number of not imported relations:
#{
@
project
.
import_failures
.
count
}
"
puts
"Total number of not imported relations:
#{
project
.
import_failures
.
count
}
"
end
end
spec/support/shared_examples/tasks/gitlab/import_export/
import_measurement
_shared_examples.rb
→
spec/support/shared_examples/tasks/gitlab/import_export/
measurable
_shared_examples.rb
View file @
492d7096
# frozen_string_literal: true
RSpec
.
shared_examples
'
import measurement
'
do
RSpec
.
shared_examples
'
measurable
'
do
context
'when measurement is enabled'
do
let
(
:measurement_enabled
)
{
true
}
...
...
spec/tasks/gitlab/import_export/export_rake_spec.rb
0 → 100644
View file @
492d7096
# frozen_string_literal: true
require
'rake_helper'
describe
'gitlab:import_export:export rake task'
do
let
(
:username
)
{
'root'
}
let
(
:namespace_path
)
{
username
}
let!
(
:user
)
{
create
(
:user
,
username:
username
)
}
let
(
:measurement_enabled
)
{
false
}
let
(
:task_params
)
{
[
username
,
namespace_path
,
project_name
,
archive_path
,
measurement_enabled
]
}
before
do
Rake
.
application
.
rake_require
(
'tasks/gitlab/import_export/export'
)
end
subject
{
run_rake_task
(
'gitlab:import_export:export'
,
task_params
)
}
context
'when project is found'
do
let
(
:project
)
{
create
(
:project
,
creator:
user
,
namespace:
user
.
namespace
)
}
let
(
:project_name
)
{
project
.
name
}
let
(
:archive_path
)
{
'spec/fixtures/gitlab/import_export/test_project_export.tar.gz'
}
around
do
|
example
|
example
.
run
ensure
File
.
delete
(
archive_path
)
end
it
'performs project export successfully'
do
expect
{
subject
}.
to
output
(
/Done!/
).
to_stdout
expect
(
File
).
to
exist
(
archive_path
)
end
it_behaves_like
'measurable'
end
end
spec/tasks/gitlab/import_export/import_rake_spec.rb
View file @
492d7096
...
...
@@ -70,7 +70,7 @@ describe 'gitlab:import_export:import rake task' do
subject
end
it_behaves_like
'
import measurement
'
it_behaves_like
'
measurable
'
end
context
'when project import is invalid'
do
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment