Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
0c3f1214
Commit
0c3f1214
authored
Nov 11, 2019
by
GitLab Bot
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add latest changes from gitlab-org/gitlab@master
parent
ff67e3ed
Changes
34
Hide whitespace changes
Inline
Side-by-side
Showing
34 changed files
with
648 additions
and
69 deletions
+648
-69
app/finders/todos_finder.rb
app/finders/todos_finder.rb
+10
-2
app/models/group.rb
app/models/group.rb
+2
-0
app/models/import_export_upload.rb
app/models/import_export_upload.rb
+1
-0
app/services/groups/import_export/export_service.rb
app/services/groups/import_export/export_service.rb
+71
-0
app/services/projects/import_export/export_service.rb
app/services/projects/import_export/export_service.rb
+1
-1
app/workers/all_queues.yml
app/workers/all_queues.yml
+1
-0
app/workers/group_export_worker.rb
app/workers/group_export_worker.rb
+15
-0
config/sidekiq_queues.yml
config/sidekiq_queues.yml
+1
-0
db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb
...e/20191111115229_add_group_id_to_import_export_uploads.rb
+9
-0
db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb
...e/20191111115431_add_group_fk_to_import_export_uploads.rb
+19
-0
db/schema.rb
db/schema.rb
+4
-1
lib/gitlab/import_export.rb
lib/gitlab/import_export.rb
+11
-3
lib/gitlab/import_export/config.rb
lib/gitlab/import_export/config.rb
+3
-2
lib/gitlab/import_export/file_importer.rb
lib/gitlab/import_export/file_importer.rb
+1
-1
lib/gitlab/import_export/group_import_export.yml
lib/gitlab/import_export/group_import_export.yml
+36
-0
lib/gitlab/import_export/group_tree_saver.rb
lib/gitlab/import_export/group_tree_saver.rb
+55
-0
lib/gitlab/import_export/project_tree_saver.rb
lib/gitlab/import_export/project_tree_saver.rb
+10
-21
lib/gitlab/import_export/reader.rb
lib/gitlab/import_export/reader.rb
+17
-10
lib/gitlab/import_export/relation_rename_service.rb
lib/gitlab/import_export/relation_rename_service.rb
+1
-1
lib/gitlab/import_export/relation_tree_saver.rb
lib/gitlab/import_export/relation_tree_saver.rb
+27
-0
lib/gitlab/import_export/saver.rb
lib/gitlab/import_export/saver.rb
+12
-6
lib/gitlab/import_export/shared.rb
lib/gitlab/import_export/shared.rb
+27
-13
spec/finders/todos_finder_spec.rb
spec/finders/todos_finder_spec.rb
+13
-0
spec/fixtures/group_export.tar.gz
spec/fixtures/group_export.tar.gz
+0
-0
spec/lib/gitlab/import_export/group_tree_saver_spec.rb
spec/lib/gitlab/import_export/group_tree_saver_spec.rb
+162
-0
spec/lib/gitlab/import_export/import_export_spec.rb
spec/lib/gitlab/import_export/import_export_spec.rb
+3
-3
spec/lib/gitlab/import_export/relation_rename_service_spec.rb
.../lib/gitlab/import_export/relation_rename_service_spec.rb
+6
-1
spec/lib/gitlab/import_export/relation_tree_saver_spec.rb
spec/lib/gitlab/import_export/relation_tree_saver_spec.rb
+42
-0
spec/lib/gitlab/import_export/saver_spec.rb
spec/lib/gitlab/import_export/saver_spec.rb
+1
-1
spec/lib/gitlab/import_export/shared_spec.rb
spec/lib/gitlab/import_export/shared_spec.rb
+1
-1
spec/services/groups/import_export/export_service_spec.rb
spec/services/groups/import_export/export_service_spec.rb
+55
-0
spec/services/import_export_clean_up_service_spec.rb
spec/services/import_export_clean_up_service_spec.rb
+1
-1
spec/services/projects/import_export/export_service_spec.rb
spec/services/projects/import_export/export_service_spec.rb
+1
-1
spec/workers/group_export_worker_spec.rb
spec/workers/group_export_worker_spec.rb
+29
-0
No files found.
app/finders/todos_finder.rb
View file @
0c3f1214
...
...
@@ -23,10 +23,16 @@ class TodosFinder
NONE
=
'0'
TODO_TYPES
=
Set
.
new
(
%w(Issue MergeRequest
Epic
)
).
freeze
TODO_TYPES
=
Set
.
new
(
%w(Issue MergeRequest)
).
freeze
attr_accessor
:current_user
,
:params
class
<<
self
def
todo_types
TODO_TYPES
end
end
def
initialize
(
current_user
,
params
=
{})
@current_user
=
current_user
@params
=
params
...
...
@@ -124,7 +130,7 @@ class TodosFinder
end
def
type?
type
.
present?
&&
TODO_TYPES
.
include?
(
type
)
type
.
present?
&&
self
.
class
.
todo_types
.
include?
(
type
)
end
def
type
...
...
@@ -201,3 +207,5 @@ class TodosFinder
end
end
end
TodosFinder
.
prepend_if_ee
(
'EE::TodosFinder'
)
app/models/group.rb
View file @
0c3f1214
...
...
@@ -55,6 +55,8 @@ class Group < Namespace
has_many
:todos
has_one
:import_export_upload
accepts_nested_attributes_for
:variables
,
allow_destroy:
true
validate
:visibility_level_allowed_by_projects
...
...
app/models/import_export_upload.rb
View file @
0c3f1214
...
...
@@ -5,6 +5,7 @@ class ImportExportUpload < ApplicationRecord
include
ObjectStorage
::
BackgroundMove
belongs_to
:project
belongs_to
:group
# These hold the project Import/Export archives (.tar.gz files)
mount_uploader
:import_file
,
ImportExportUploader
...
...
app/services/groups/import_export/export_service.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
module
Groups
module
ImportExport
class
ExportService
def
initialize
(
group
:,
user
:,
params:
{})
@group
=
group
@current_user
=
user
@params
=
params
@shared
=
@params
[
:shared
]
||
Gitlab
::
ImportExport
::
Shared
.
new
(
@group
)
end
def
execute
save!
end
private
attr_accessor
:shared
def
save!
if
savers
.
all?
(
&
:save
)
notify_success
else
cleanup_and_notify_error!
end
end
def
savers
[
tree_exporter
,
file_saver
]
end
def
tree_exporter
Gitlab
::
ImportExport
::
GroupTreeSaver
.
new
(
group:
@group
,
current_user:
@current_user
,
shared:
@shared
,
params:
@params
)
end
def
file_saver
Gitlab
::
ImportExport
::
Saver
.
new
(
exportable:
@group
,
shared:
@shared
)
end
def
cleanup_and_notify_error
FileUtils
.
rm_rf
(
shared
.
export_path
)
notify_error
end
def
cleanup_and_notify_error!
cleanup_and_notify_error
raise
Gitlab
::
ImportExport
::
Error
.
new
(
shared
.
errors
.
to_sentence
)
end
def
notify_success
@shared
.
logger
.
info
(
group_id:
@group
.
id
,
group_name:
@group
.
name
,
message:
'Group Import/Export: Export succeeded'
)
end
def
notify_error
@shared
.
logger
.
error
(
group_id:
@group
.
id
,
group_name:
@group
.
name
,
error:
@shared
.
errors
.
join
(
', '
),
message:
'Group Import/Export: Export failed'
)
end
end
end
end
app/services/projects/import_export/export_service.rb
View file @
0c3f1214
...
...
@@ -24,7 +24,7 @@ module Projects
def
save_all!
if
save_exporters
Gitlab
::
ImportExport
::
Saver
.
save
(
project
:
project
,
shared:
shared
)
Gitlab
::
ImportExport
::
Saver
.
save
(
exportable
:
project
,
shared:
shared
)
notify_success
else
cleanup_and_notify_error!
...
...
app/workers/all_queues.yml
View file @
0c3f1214
...
...
@@ -179,3 +179,4 @@
-
import_issues_csv
-
project_daily_statistics
-
create_evidence
-
group_export
app/workers/group_export_worker.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
class
GroupExportWorker
include
ApplicationWorker
include
ExceptionBacktrace
feature_category
:source_code_management
def
perform
(
current_user_id
,
group_id
,
params
=
{})
current_user
=
User
.
find
(
current_user_id
)
group
=
Group
.
find
(
group_id
)
::
Groups
::
ImportExport
::
ExportService
.
new
(
group:
group
,
user:
current_user
,
params:
params
).
execute
end
end
config/sidekiq_queues.yml
View file @
0c3f1214
...
...
@@ -98,6 +98,7 @@
- [update_namespace_statistics, 1]
- [chaos, 2]
- [create_evidence, 2]
- [group_export, 1]
# EE-specific queues
- [analytics, 1]
...
...
db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
class
AddGroupIdToImportExportUploads
<
ActiveRecord
::
Migration
[
5.2
]
DOWNTIME
=
false
def
change
add_column
:import_export_uploads
,
:group_id
,
:bigint
end
end
db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
class
AddGroupFkToImportExportUploads
<
ActiveRecord
::
Migration
[
5.2
]
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
disable_ddl_transaction!
def
up
add_concurrent_foreign_key
:import_export_uploads
,
:namespaces
,
column: :group_id
,
on_delete: :cascade
add_concurrent_index
:import_export_uploads
,
:group_id
,
unique:
true
,
where:
'group_id IS NOT NULL'
end
def
down
remove_foreign_key_without_error
(
:import_export_uploads
,
column: :group_id
)
remove_concurrent_index
(
:import_export_uploads
,
:group_id
)
end
end
db/schema.rb
View file @
0c3f1214
...
...
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord
::
Schema
.
define
(
version:
2019_11_
05_140942
)
do
ActiveRecord
::
Schema
.
define
(
version:
2019_11_
11_115431
)
do
# These are extensions that must be enabled in order to support this database
enable_extension
"pg_trgm"
...
...
@@ -1874,6 +1874,8 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
t
.
integer
"project_id"
t
.
text
"import_file"
t
.
text
"export_file"
t
.
bigint
"group_id"
t
.
index
[
"group_id"
],
name:
"index_import_export_uploads_on_group_id"
,
unique:
true
,
where:
"(group_id IS NOT NULL)"
t
.
index
[
"project_id"
],
name:
"index_import_export_uploads_on_project_id"
t
.
index
[
"updated_at"
],
name:
"index_import_export_uploads_on_updated_at"
end
...
...
@@ -4288,6 +4290,7 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
add_foreign_key
"group_group_links"
,
"namespaces"
,
column:
"shared_group_id"
,
on_delete: :cascade
add_foreign_key
"group_group_links"
,
"namespaces"
,
column:
"shared_with_group_id"
,
on_delete: :cascade
add_foreign_key
"identities"
,
"saml_providers"
,
name:
"fk_aade90f0fc"
,
on_delete: :cascade
add_foreign_key
"import_export_uploads"
,
"namespaces"
,
column:
"group_id"
,
name:
"fk_83319d9721"
,
on_delete: :cascade
add_foreign_key
"import_export_uploads"
,
"projects"
,
on_delete: :cascade
add_foreign_key
"index_statuses"
,
"projects"
,
name:
"fk_74b2492545"
,
on_delete: :cascade
add_foreign_key
"insights"
,
"namespaces"
,
on_delete: :cascade
...
...
lib/gitlab/import_export.rb
View file @
0c3f1214
...
...
@@ -15,7 +15,7 @@ module Gitlab
end
def
storage_path
File
.
join
(
Settings
.
shared
[
'path'
],
'tmp/
project
_exports'
)
File
.
join
(
Settings
.
shared
[
'path'
],
'tmp/
gitlab
_exports'
)
end
def
import_upload_path
(
filename
:)
...
...
@@ -50,8 +50,8 @@ module Gitlab
'VERSION'
end
def
export_filename
(
project
:)
basename
=
"
#{
Time
.
now
.
strftime
(
'%Y-%m-%d_%H-%M-%3N'
)
}
_
#{
project
.
full_path
.
tr
(
'/'
,
'_'
)
}
"
def
export_filename
(
exportable
:)
basename
=
"
#{
Time
.
now
.
strftime
(
'%Y-%m-%d_%H-%M-%3N'
)
}
_
#{
exportable
.
full_path
.
tr
(
'/'
,
'_'
)
}
"
"
#{
basename
[
0
..
FILENAME_LIMIT
]
}
_export.tar.gz"
end
...
...
@@ -63,6 +63,14 @@ module Gitlab
def
reset_tokens?
true
end
def
group_filename
'group.json'
end
def
group_config_file
Rails
.
root
.
join
(
'lib/gitlab/import_export/group_import_export.yml'
)
end
end
end
...
...
lib/gitlab/import_export/config.rb
View file @
0c3f1214
...
...
@@ -3,7 +3,8 @@
module
Gitlab
module
ImportExport
class
Config
def
initialize
def
initialize
(
config:
Gitlab
::
ImportExport
.
config_file
)
@config
=
config
@hash
=
parse_yaml
@hash
.
deep_symbolize_keys!
@ee_hash
=
@hash
.
delete
(
:ee
)
||
{}
...
...
@@ -50,7 +51,7 @@ module Gitlab
end
def
parse_yaml
YAML
.
load_file
(
Gitlab
::
ImportExport
.
config_file
)
YAML
.
load_file
(
@config
)
end
end
end
...
...
lib/gitlab/import_export/file_importer.rb
View file @
0c3f1214
...
...
@@ -60,7 +60,7 @@ module Gitlab
def
copy_archive
return
if
@archive_file
@archive_file
=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project
:
@project
))
@archive_file
=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
exportable
:
@project
))
download_or_copy_upload
(
@project
.
import_export_upload
.
import_file
,
@archive_file
)
end
...
...
lib/gitlab/import_export/group_import_export.yml
0 → 100644
View file @
0c3f1214
# Model relationships to be included in the group import/export
#
# This list _must_ only contain relationships that are available to both FOSS and
# Enterprise editions. EE specific relationships must be defined in the `ee` section further
# down below.
tree
:
group
:
-
:milestones
-
:badges
-
labels
:
-
:priorities
-
:boards
-
members
:
-
:user
included_attributes
:
excluded_attributes
:
group
:
-
:runners_token
-
:runners_token_encrypted
methods
:
labels
:
-
:type
badges
:
-
:type
preloads
:
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee
:
tree
:
group
:
-
:epics
lib/gitlab/import_export/group_tree_saver.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
module
Gitlab
module
ImportExport
class
GroupTreeSaver
attr_reader
:full_path
def
initialize
(
group
:,
current_user
:,
shared
:,
params:
{})
@params
=
params
@current_user
=
current_user
@shared
=
shared
@group
=
group
@full_path
=
File
.
join
(
@shared
.
export_path
,
ImportExport
.
group_filename
)
end
def
save
group_tree
=
serialize
(
@group
,
reader
.
group_tree
)
tree_saver
.
save
(
group_tree
,
@shared
.
export_path
,
ImportExport
.
group_filename
)
true
rescue
=>
e
@shared
.
error
(
e
)
false
end
private
def
serialize
(
group
,
relations_tree
)
group_tree
=
tree_saver
.
serialize
(
group
,
relations_tree
)
group
.
descendants
.
each
do
|
descendant
|
group_tree
[
'descendants'
]
=
[]
unless
group_tree
[
'descendants'
]
group_tree
[
'descendants'
]
<<
serialize
(
descendant
,
relations_tree
)
end
group_tree
rescue
=>
e
@shared
.
error
(
e
)
end
def
reader
@reader
||=
Gitlab
::
ImportExport
::
Reader
.
new
(
shared:
@shared
,
config:
Gitlab
::
ImportExport
::
Config
.
new
(
config:
Gitlab
::
ImportExport
.
group_config_file
).
to_h
)
end
def
tree_saver
@tree_saver
||=
RelationTreeSaver
.
new
end
end
end
end
lib/gitlab/import_export/project_tree_saver.rb
View file @
0c3f1214
...
...
@@ -3,25 +3,20 @@
module
Gitlab
module
ImportExport
class
ProjectTreeSaver
include
Gitlab
::
ImportExport
::
CommandLineUtil
attr_reader
:full_path
def
initialize
(
project
:,
current_user
:,
shared
:,
params:
{})
@params
=
params
@project
=
project
@params
=
params
@project
=
project
@current_user
=
current_user
@shared
=
shared
@full_path
=
File
.
join
(
@shared
.
export_path
,
ImportExport
.
project_filename
)
@shared
=
shared
@full_path
=
File
.
join
(
@shared
.
export_path
,
ImportExport
.
project_filename
)
end
def
save
mkdir_p
(
@shared
.
export_path
)
project_tree
=
serialize_project_tree
project_tree
=
tree_saver
.
serialize
(
@project
,
reader
.
project_tree
)
fix_project_tree
(
project_tree
)
project_tree_json
=
JSON
.
generate
(
project_tree
)
File
.
write
(
full_path
,
project_tree_json
)
tree_saver
.
save
(
project_tree
,
@shared
.
export_path
,
ImportExport
.
project_filename
)
true
rescue
=>
e
...
...
@@ -43,16 +38,6 @@ module Gitlab
RelationRenameService
.
add_new_associations
(
project_tree
)
end
def
serialize_project_tree
if
Feature
.
enabled?
(
:export_fast_serialize
,
default_enabled:
true
)
Gitlab
::
ImportExport
::
FastHashSerializer
.
new
(
@project
,
reader
.
project_tree
)
.
execute
else
@project
.
as_json
(
reader
.
project_tree
)
end
end
def
reader
@reader
||=
Gitlab
::
ImportExport
::
Reader
.
new
(
shared:
@shared
)
end
...
...
@@ -74,6 +59,10 @@ module Gitlab
GroupMembersFinder
.
new
(
@project
.
group
).
execute
.
where
.
not
(
user_id:
non_null_user_ids
)
end
def
tree_saver
@tree_saver
||=
RelationTreeSaver
.
new
end
end
end
end
lib/gitlab/import_export/reader.rb
View file @
0c3f1214
...
...
@@ -5,24 +5,31 @@ module Gitlab
class
Reader
attr_reader
:tree
,
:attributes_finder
def
initialize
(
shared
:)
@shared
=
shared
@attributes_finder
=
Gitlab
::
ImportExport
::
AttributesFinder
.
new
(
config:
ImportExport
::
Config
.
new
.
to_h
)
def
initialize
(
shared
:,
config:
ImportExport
::
Config
.
new
.
to_h
)
@shared
=
shared
@config
=
config
@attributes_finder
=
Gitlab
::
ImportExport
::
AttributesFinder
.
new
(
config:
@config
)
end
# Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
# for outputting a project in JSON format, including its relations and sub relations.
def
project_tree
attributes_finder
.
find_root
(
:project
)
rescue
=>
e
@shared
.
error
(
e
)
false
tree_by_key
(
:project
)
end
def
group_tree
tree_by_key
(
:group
)
end
def
group_members_tree
attributes_finder
.
find_root
(
:group_members
)
tree_by_key
(
:group_members
)
end
def
tree_by_key
(
key
)
attributes_finder
.
find_root
(
key
)
rescue
=>
e
@shared
.
error
(
e
)
false
end
end
end
...
...
lib/gitlab/import_export/relation_rename_service.rb
View file @
0c3f1214
...
...
@@ -8,7 +8,7 @@
# The behavior of these renamed relationships should be transient and it should
# only last one release until you completely remove the renaming from the list.
#
# When importing, this class will check the
project
hash and:
# When importing, this class will check the hash and:
# - if only the old relationship name is found, it will rename it with the new one
# - if only the new relationship name is found, it will do nothing
# - if it finds both, it will use the new relationship data
...
...
lib/gitlab/import_export/relation_tree_saver.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
module
Gitlab
module
ImportExport
class
RelationTreeSaver
include
Gitlab
::
ImportExport
::
CommandLineUtil
def
serialize
(
exportable
,
relations_tree
)
if
Feature
.
enabled?
(
:export_fast_serialize
,
default_enabled:
true
)
Gitlab
::
ImportExport
::
FastHashSerializer
.
new
(
exportable
,
relations_tree
)
.
execute
else
exportable
.
as_json
(
relations_tree
)
end
end
def
save
(
tree
,
dir_path
,
filename
)
mkdir_p
(
dir_path
)
tree_json
=
JSON
.
generate
(
tree
)
File
.
write
(
File
.
join
(
dir_path
,
filename
),
tree_json
)
end
end
end
end
lib/gitlab/import_export/saver.rb
View file @
0c3f1214
...
...
@@ -9,16 +9,16 @@ module Gitlab
new
(
*
args
).
save
end
def
initialize
(
project
:,
shared
:)
@
project
=
project
@shared
=
shared
def
initialize
(
exportable
:,
shared
:)
@
exportable
=
exportable
@shared
=
shared
end
def
save
if
compress_and_save
remove_export_path
Rails
.
logger
.
info
(
"Saved
project
export
#{
archive_file
}
"
)
# rubocop:disable Gitlab/RailsLogger
Rails
.
logger
.
info
(
"Saved
#{
@exportable
.
class
}
export
#{
archive_file
}
"
)
# rubocop:disable Gitlab/RailsLogger
save_upload
else
...
...
@@ -48,11 +48,11 @@ module Gitlab
end
def
archive_file
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project:
@project
))
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
exportable:
@exportable
))
end
def
save_upload
upload
=
ImportExportUpload
.
find_or_initialize_by
(
project:
@project
)
upload
=
initialize_upload
File
.
open
(
archive_file
)
{
|
file
|
upload
.
export_file
=
file
}
...
...
@@ -62,6 +62,12 @@ module Gitlab
def
error_message
"Unable to save
#{
archive_file
}
into
#{
@shared
.
export_path
}
."
end
def
initialize_upload
exportable_kind
=
@exportable
.
class
.
name
.
downcase
ImportExportUpload
.
find_or_initialize_by
(
Hash
[
exportable_kind
,
@exportable
])
end
end
end
end
lib/gitlab/import_export/shared.rb
View file @
0c3f1214
...
...
@@ -23,21 +23,21 @@
module
Gitlab
module
ImportExport
class
Shared
attr_reader
:errors
,
:
project
attr_reader
:errors
,
:
exportable
,
:logger
LOCKS_DIRECTORY
=
'locks'
def
initialize
(
project
)
@
project
=
project
@errors
=
[]
@logger
=
Gitlab
::
Import
::
Logger
.
build
def
initialize
(
exportable
)
@
exportable
=
exportable
@errors
=
[]
@logger
=
Gitlab
::
Import
::
Logger
.
build
end
def
active_export_count
Dir
[
File
.
join
(
base_path
,
'*'
)].
count
{
|
name
|
File
.
basename
(
name
)
!=
LOCKS_DIRECTORY
&&
File
.
directory?
(
name
)
}
end
# The path where the
project metadata and repository bundle
is saved
# The path where the
exportable metadata and repository bundle (in case of project)
is saved
def
export_path
@export_path
||=
Gitlab
::
ImportExport
.
export_path
(
relative_path:
relative_path
)
end
...
...
@@ -84,11 +84,18 @@ module Gitlab
end
def
relative_archive_path
@relative_archive_path
||=
File
.
join
(
@project
.
disk
_path
,
SecureRandom
.
hex
)
@relative_archive_path
||=
File
.
join
(
relative_base
_path
,
SecureRandom
.
hex
)
end
def
relative_base_path
@project
.
disk_path
case
exportable_type
when
'Project'
@exportable
.
disk_path
when
'Group'
@exportable
.
full_path
else
raise
Gitlab
::
ImportExport
::
Error
.
new
(
"Unsupported Exportable Type
#{
@exportable
&
.
class
}
"
)
end
end
def
log_error
(
details
)
...
...
@@ -100,17 +107,24 @@ module Gitlab
end
def
log_base_data
{
importer:
'Import/Export'
,
import_jid:
@project
&
.
import_state
&
.
jid
,
project_id:
@project
&
.
id
,
project_path:
@project
&
.
full_path
log
=
{
importer:
'Import/Export'
,
exportable_id:
@exportable
&
.
id
,
exportable_path:
@exportable
&
.
full_path
}
log
[
:import_jid
]
=
@exportable
&
.
import_state
&
.
jid
if
exportable_type
==
'Project'
log
end
def
filtered_error_message
(
message
)
Projects
::
ImportErrorFilter
.
filter_message
(
message
)
end
def
exportable_type
@exportable
.
class
.
name
end
end
end
end
spec/finders/todos_finder_spec.rb
View file @
0c3f1214
...
...
@@ -234,6 +234,19 @@ describe TodosFinder do
end
end
describe
'.todo_types'
do
it
'returns the expected types'
do
expected_result
=
if
Gitlab
.
ee?
%w[Epic Issue MergeRequest]
else
%w[Issue MergeRequest]
end
expect
(
described_class
.
todo_types
).
to
contain_exactly
(
*
expected_result
)
end
end
describe
'#any_for_target?'
do
it
'returns true if there are any todos for the given target'
do
todo
=
create
(
:todo
,
:pending
)
...
...
spec/fixtures/group_export.tar.gz
0 → 100644
View file @
0c3f1214
File added
spec/lib/gitlab/import_export/group_tree_saver_spec.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
GroupTreeSaver
do
describe
'saves the group tree into a json object'
do
let
(
:shared
)
{
Gitlab
::
ImportExport
::
Shared
.
new
(
group
)
}
let
(
:group_tree_saver
)
{
described_class
.
new
(
group:
group
,
current_user:
user
,
shared:
shared
)
}
let
(
:export_path
)
{
"
#{
Dir
.
tmpdir
}
/group_tree_saver_spec"
}
let
(
:user
)
{
create
(
:user
)
}
let!
(
:group
)
{
setup_group
}
before
do
group
.
add_maintainer
(
user
)
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
end
after
do
FileUtils
.
rm_rf
(
export_path
)
end
it
'saves group successfully'
do
expect
(
group_tree_saver
.
save
).
to
be
true
end
context
':export_fast_serialize feature flag checks'
do
before
do
expect
(
Gitlab
::
ImportExport
::
Reader
).
to
receive
(
:new
).
with
(
shared:
shared
,
config:
group_config
).
and_return
(
reader
)
expect
(
reader
).
to
receive
(
:group_tree
).
and_return
(
group_tree
)
end
let
(
:reader
)
{
instance_double
(
'Gitlab::ImportExport::Reader'
)
}
let
(
:group_config
)
{
Gitlab
::
ImportExport
::
Config
.
new
(
config:
Gitlab
::
ImportExport
.
group_config_file
).
to_h
}
let
(
:group_tree
)
do
{
include:
[{
milestones:
{
include:
[]
}
}],
preload:
{
milestones:
nil
}
}
end
context
'when :export_fast_serialize feature is enabled'
do
before
do
stub_feature_flags
(
export_fast_serialize:
true
)
end
it
'uses FastHashSerializer'
do
expect_any_instance_of
(
Gitlab
::
ImportExport
::
FastHashSerializer
).
to
receive
(
:execute
).
and_call_original
group_tree_saver
.
save
end
end
context
'when :export_fast_serialize feature is disabled'
do
before
do
stub_feature_flags
(
export_fast_serialize:
false
)
end
it
'is serialized via built-in `as_json`'
do
expect
(
group
).
to
receive
(
:as_json
).
with
(
group_tree
).
and_call_original
group_tree_saver
.
save
end
end
end
# It is mostly duplicated in
# `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb`
# except:
# context 'with description override' do
# context 'group members' do
# ^ These are specific for the groupTreeSaver
context
'JSON'
do
let
(
:saved_group_json
)
do
group_tree_saver
.
save
group_json
(
group_tree_saver
.
full_path
)
end
it
'saves the correct json'
do
expect
(
saved_group_json
).
to
include
({
'description'
=>
'description'
,
'visibility_level'
=>
20
})
end
it
'has milestones'
do
expect
(
saved_group_json
[
'milestones'
]).
not_to
be_empty
end
it
'has labels'
do
expect
(
saved_group_json
[
'labels'
]).
not_to
be_empty
end
it
'has boards'
do
expect
(
saved_group_json
[
'boards'
]).
not_to
be_empty
end
it
'has group members'
do
expect
(
saved_group_json
[
'members'
]).
not_to
be_empty
end
it
'has priorities associated to labels'
do
expect
(
saved_group_json
[
'labels'
].
first
[
'priorities'
]).
not_to
be_empty
end
it
'has badges'
do
expect
(
saved_group_json
[
'badges'
]).
not_to
be_empty
end
context
'group members'
do
let
(
:user2
)
{
create
(
:user
,
email:
'group@member.com'
)
}
let
(
:member_emails
)
do
saved_group_json
[
'members'
].
map
do
|
pm
|
pm
[
'user'
][
'email'
]
end
end
before
do
group
.
add_developer
(
user2
)
end
it
'exports group members as group owner'
do
group
.
add_owner
(
user
)
expect
(
member_emails
).
to
include
(
'group@member.com'
)
end
context
'as admin'
do
let
(
:user
)
{
create
(
:admin
)
}
it
'exports group members as admin'
do
expect
(
member_emails
).
to
include
(
'group@member.com'
)
end
it
'exports group members'
do
member_types
=
saved_group_json
[
'members'
].
map
{
|
pm
|
pm
[
'source_type'
]
}
expect
(
member_types
).
to
all
(
eq
(
'Namespace'
))
end
end
end
context
'group attributes'
do
it
'does not contain the runners token'
do
expect
(
saved_group_json
).
not_to
include
(
"runners_token"
=>
'token'
)
end
end
end
end
def
setup_group
group
=
create
(
:group
,
description:
'description'
)
create
(
:milestone
,
group:
group
)
create
(
:group_badge
,
group:
group
)
group_label
=
create
(
:group_label
,
group:
group
)
create
(
:label_priority
,
label:
group_label
,
priority:
1
)
create
(
:board
,
group:
group
)
create
(
:group_badge
,
group:
group
)
group
end
def
group_json
(
filename
)
JSON
.
parse
(
IO
.
read
(
filename
))
end
end
spec/lib/gitlab/import_export/import_export_spec.rb
View file @
0c3f1214
...
...
@@ -6,17 +6,17 @@ describe Gitlab::ImportExport do
let
(
:project
)
{
create
(
:project
,
:public
,
path:
'project-path'
,
namespace:
group
)
}
it
'contains the project path'
do
expect
(
described_class
.
export_filename
(
project
:
project
)).
to
include
(
project
.
path
)
expect
(
described_class
.
export_filename
(
exportable
:
project
)).
to
include
(
project
.
path
)
end
it
'contains the namespace path'
do
expect
(
described_class
.
export_filename
(
project
:
project
)).
to
include
(
project
.
namespace
.
full_path
.
tr
(
'/'
,
'_'
))
expect
(
described_class
.
export_filename
(
exportable
:
project
)).
to
include
(
project
.
namespace
.
full_path
.
tr
(
'/'
,
'_'
))
end
it
'does not go over a certain length'
do
project
.
path
=
'a'
*
100
expect
(
described_class
.
export_filename
(
project
:
project
).
length
).
to
be
<
70
expect
(
described_class
.
export_filename
(
exportable
:
project
).
length
).
to
be
<
70
end
end
end
spec/lib/gitlab/import_export/relation_rename_service_spec.rb
View file @
0c3f1214
...
...
@@ -96,15 +96,20 @@ describe Gitlab::ImportExport::RelationRenameService do
let
(
:export_content_path
)
{
project_tree_saver
.
full_path
}
let
(
:export_content_hash
)
{
ActiveSupport
::
JSON
.
decode
(
File
.
read
(
export_content_path
))
}
let
(
:injected_hash
)
{
renames
.
values
.
product
([{}]).
to_h
}
let
(
:relation_tree_saver
)
{
Gitlab
::
ImportExport
::
RelationTreeSaver
.
new
}
let
(
:project_tree_saver
)
do
Gitlab
::
ImportExport
::
ProjectTreeSaver
.
new
(
project:
project
,
current_user:
user
,
shared:
shared
)
end
before
do
allow
(
project_tree_saver
).
to
receive
(
:tree_saver
).
and_return
(
relation_tree_saver
)
end
it
'adds old relationships to the exported file'
do
# we inject relations with new names that should be rewritten
expect
(
project_tree_saver
).
to
receive
(
:serialize_project_tre
e
).
and_wrap_original
do
|
method
,
*
args
|
expect
(
relation_tree_saver
).
to
receive
(
:serializ
e
).
and_wrap_original
do
|
method
,
*
args
|
method
.
call
(
*
args
).
merge
(
injected_hash
)
end
...
...
spec/lib/gitlab/import_export/relation_tree_saver_spec.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
RelationTreeSaver
do
let
(
:exportable
)
{
create
(
:group
)
}
let
(
:relation_tree_saver
)
{
described_class
.
new
}
let
(
:tree
)
{
{}
}
describe
'#serialize'
do
context
'when :export_fast_serialize feature is enabled'
do
let
(
:serializer
)
{
instance_double
(
Gitlab
::
ImportExport
::
FastHashSerializer
)
}
before
do
stub_feature_flags
(
export_fast_serialize:
true
)
end
it
'uses FastHashSerializer'
do
expect
(
Gitlab
::
ImportExport
::
FastHashSerializer
)
.
to
receive
(
:new
)
.
with
(
exportable
,
tree
)
.
and_return
(
serializer
)
expect
(
serializer
).
to
receive
(
:execute
)
relation_tree_saver
.
serialize
(
exportable
,
tree
)
end
end
context
'when :export_fast_serialize feature is disabled'
do
before
do
stub_feature_flags
(
export_fast_serialize:
false
)
end
it
'is serialized via built-in `as_json`'
do
expect
(
exportable
).
to
receive
(
:as_json
).
with
(
tree
)
relation_tree_saver
.
serialize
(
exportable
,
tree
)
end
end
end
end
spec/lib/gitlab/import_export/saver_spec.rb
View file @
0c3f1214
...
...
@@ -5,7 +5,7 @@ describe Gitlab::ImportExport::Saver do
let!
(
:project
)
{
create
(
:project
,
:public
,
name:
'project'
)
}
let
(
:export_path
)
{
"
#{
Dir
.
tmpdir
}
/project_tree_saver_spec"
}
let
(
:shared
)
{
project
.
import_export_shared
}
subject
{
described_class
.
new
(
project
:
project
,
shared:
shared
)
}
subject
{
described_class
.
new
(
exportable
:
project
,
shared:
shared
)
}
before
do
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
...
...
spec/lib/gitlab/import_export/shared_spec.rb
View file @
0c3f1214
...
...
@@ -7,7 +7,7 @@ describe Gitlab::ImportExport::Shared do
context
'with a repository on disk'
do
let
(
:project
)
{
create
(
:project
,
:repository
)
}
let
(
:base_path
)
{
%(/tmp/
project
_exports/#{project.disk_path}/)
}
let
(
:base_path
)
{
%(/tmp/
gitlab
_exports/#{project.disk_path}/)
}
describe
'#archive_path'
do
it
'uses a random hash to avoid conflicts'
do
...
...
spec/services/groups/import_export/export_service_spec.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
require
'spec_helper'
describe
Groups
::
ImportExport
::
ExportService
do
describe
'#execute'
do
let!
(
:user
)
{
create
(
:user
)
}
let
(
:group
)
{
create
(
:group
)
}
let
(
:shared
)
{
Gitlab
::
ImportExport
::
Shared
.
new
(
group
)
}
let
(
:export_path
)
{
shared
.
export_path
}
let
(
:service
)
{
described_class
.
new
(
group:
group
,
user:
user
,
params:
{
shared:
shared
})
}
after
do
FileUtils
.
rm_rf
(
export_path
)
end
it
'saves the models'
do
expect
(
Gitlab
::
ImportExport
::
GroupTreeSaver
).
to
receive
(
:new
).
and_call_original
service
.
execute
end
context
'when saver succeeds'
do
it
'saves the group in the file system'
do
service
.
execute
expect
(
group
.
import_export_upload
.
export_file
.
file
).
not_to
be_nil
expect
(
File
.
directory?
(
export_path
)).
to
eq
(
false
)
expect
(
File
.
exist?
(
shared
.
archive_path
)).
to
eq
(
false
)
end
end
context
'when saving services fail'
do
before
do
allow
(
service
).
to
receive_message_chain
(
:tree_exporter
,
:save
).
and_return
(
false
)
end
it
'removes the remaining exported data'
do
allow_any_instance_of
(
Gitlab
::
ImportExport
::
Saver
).
to
receive
(
:compress_and_save
).
and_return
(
false
)
expect
{
service
.
execute
}.
to
raise_error
(
Gitlab
::
ImportExport
::
Error
)
expect
(
group
.
import_export_upload
).
to
be_nil
expect
(
File
.
directory?
(
export_path
)).
to
eq
(
false
)
expect
(
File
.
exist?
(
shared
.
archive_path
)).
to
eq
(
false
)
end
it
'notifies logger'
do
expect_any_instance_of
(
Gitlab
::
Import
::
Logger
).
to
receive
(
:error
)
expect
{
service
.
execute
}.
to
raise_error
(
Gitlab
::
ImportExport
::
Error
)
end
end
end
end
spec/services/import_export_clean_up_service_spec.rb
View file @
0c3f1214
...
...
@@ -6,7 +6,7 @@ describe ImportExportCleanUpService do
describe
'#execute'
do
let
(
:service
)
{
described_class
.
new
}
let
(
:tmp_import_export_folder
)
{
'tmp/
project
_exports'
}
let
(
:tmp_import_export_folder
)
{
'tmp/
gitlab
_exports'
}
context
'when the import/export directory does not exist'
do
it
'does not remove any archives'
do
...
...
spec/services/projects/import_export/export_service_spec.rb
View file @
0c3f1214
...
...
@@ -66,7 +66,7 @@ describe Projects::ImportExport::ExportService do
end
it
'saves the project in the file system'
do
expect
(
Gitlab
::
ImportExport
::
Saver
).
to
receive
(
:save
).
with
(
project
:
project
,
shared:
shared
)
expect
(
Gitlab
::
ImportExport
::
Saver
).
to
receive
(
:save
).
with
(
exportable
:
project
,
shared:
shared
)
service
.
execute
end
...
...
spec/workers/group_export_worker_spec.rb
0 → 100644
View file @
0c3f1214
# frozen_string_literal: true
require
'spec_helper'
describe
GroupExportWorker
do
let!
(
:user
)
{
create
(
:user
)
}
let!
(
:group
)
{
create
(
:group
)
}
subject
{
described_class
.
new
}
describe
'#perform'
do
context
'when it succeeds'
do
it
'calls the ExportService'
do
expect_any_instance_of
(
::
Groups
::
ImportExport
::
ExportService
).
to
receive
(
:execute
)
subject
.
perform
(
user
.
id
,
group
.
id
,
{})
end
end
context
'when it fails'
do
it
'raises an exception when params are invalid'
do
expect_any_instance_of
(
::
Groups
::
ImportExport
::
ExportService
).
not_to
receive
(
:execute
)
expect
{
subject
.
perform
(
1234
,
group
.
id
,
{})
}.
to
raise_exception
(
ActiveRecord
::
RecordNotFound
)
expect
{
subject
.
perform
(
user
.
id
,
1234
,
{})
}.
to
raise_exception
(
ActiveRecord
::
RecordNotFound
)
end
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment