Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Boxiang Sun
gitlab-ce
Commits
a2bf1641
Commit
a2bf1641
authored
Jun 25, 2018
by
James Lopez
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Update Import/Export to use object storage (based on aa feature flag)
parent
b0fa01fc
Changes
31
Hide whitespace changes
Inline
Side-by-side
Showing
31 changed files
with
508 additions
and
45 deletions
+508
-45
app/controllers/projects_controller.rb
app/controllers/projects_controller.rb
+12
-5
app/models/import_export_upload.rb
app/models/import_export_upload.rb
+13
-0
app/models/project.rb
app/models/project.rb
+13
-7
app/services/import_export_clean_up_service.rb
app/services/import_export_clean_up_service.rb
+10
-1
app/uploaders/import_export_uploader.rb
app/uploaders/import_export_uploader.rb
+15
-0
app/views/projects/_export.html.haml
app/views/projects/_export.html.haml
+1
-1
changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml
...46246-gitlab-project-export-should-use-object-storage.yml
+5
-0
db/migrate/20180625113853_create_import_export_uploads.rb
db/migrate/20180625113853_create_import_export_uploads.rb
+16
-0
db/schema.rb
db/schema.rb
+10
-0
doc/administration/raketasks/project_import_export.md
doc/administration/raketasks/project_import_export.md
+7
-0
lib/api/project_export.rb
lib/api/project_export.rb
+7
-3
lib/gitlab/import_export.rb
lib/gitlab/import_export.rb
+4
-0
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
...ort/after_export_strategies/base_after_export_strategy.rb
+12
-4
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
...ort_export/after_export_strategies/web_upload_strategy.rb
+20
-6
lib/gitlab/import_export/saver.rb
lib/gitlab/import_export/saver.rb
+29
-2
spec/controllers/projects_controller_spec.rb
spec/controllers/projects_controller_spec.rb
+41
-9
spec/factories/import_export_uploads.rb
spec/factories/import_export_uploads.rb
+5
-0
spec/factories/projects.rb
spec/factories/projects.rb
+16
-0
spec/features/projects/import_export/export_file_spec.rb
spec/features/projects/import_export/export_file_spec.rb
+1
-0
spec/features/projects/import_export/namespace_export_file_spec.rb
...ures/projects/import_export/namespace_export_file_spec.rb
+1
-0
spec/fixtures/project_export.tar.gz
spec/fixtures/project_export.tar.gz
+0
-0
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
...ategies/base_after_export_strategy_object_storage_spec.rb
+105
-0
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
...fter_export_strategies/base_after_export_strategy_spec.rb
+1
-0
spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
...xport/after_export_strategies/web_upload_strategy_spec.rb
+26
-5
spec/lib/gitlab/import_export/all_models.yml
spec/lib/gitlab/import_export/all_models.yml
+1
-0
spec/lib/gitlab/import_export/saver_spec.rb
spec/lib/gitlab/import_export/saver_spec.rb
+43
-0
spec/models/import_export_upload_spec.rb
spec/models/import_export_upload_spec.rb
+25
-0
spec/models/project_spec.rb
spec/models/project_spec.rb
+8
-2
spec/requests/api/project_export_spec.rb
spec/requests/api/project_export_spec.rb
+23
-0
spec/services/import_export_clean_up_service_spec.rb
spec/services/import_export_clean_up_service_spec.rb
+18
-0
spec/uploaders/import_export_uploader_spec.rb
spec/uploaders/import_export_uploader_spec.rb
+20
-0
No files found.
app/controllers/projects_controller.rb
View file @
a2bf1641
...
@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController
...
@@ -2,6 +2,7 @@ class ProjectsController < Projects::ApplicationController
include
IssuableCollections
include
IssuableCollections
include
ExtractsPath
include
ExtractsPath
include
PreviewMarkdown
include
PreviewMarkdown
include
SendFileUpload
before_action
:whitelist_query_limiting
,
only:
[
:create
]
before_action
:whitelist_query_limiting
,
only:
[
:create
]
before_action
:authenticate_user!
,
except:
[
:index
,
:show
,
:activity
,
:refs
]
before_action
:authenticate_user!
,
except:
[
:index
,
:show
,
:activity
,
:refs
]
...
@@ -188,9 +189,9 @@ class ProjectsController < Projects::ApplicationController
...
@@ -188,9 +189,9 @@ class ProjectsController < Projects::ApplicationController
end
end
def
download_export
def
download_export
export_project_path
=
@project
.
export_project_path
if
export_project_object_storage?
send_upload
(
@project
.
import_export_upload
.
export_file
)
if
export_project_path
els
if
export_project_path
send_file
export_project_path
,
disposition:
'attachment'
send_file
export_project_path
,
disposition:
'attachment'
else
else
redirect_to
(
redirect_to
(
...
@@ -265,8 +266,6 @@ class ProjectsController < Projects::ApplicationController
...
@@ -265,8 +266,6 @@ class ProjectsController < Projects::ApplicationController
render
json:
options
.
to_json
render
json:
options
.
to_json
end
end
private
# Render project landing depending of which features are available
# Render project landing depending of which features are available
# So if page is not availble in the list it renders the next page
# So if page is not availble in the list it renders the next page
#
#
...
@@ -424,4 +423,12 @@ class ProjectsController < Projects::ApplicationController
...
@@ -424,4 +423,12 @@ class ProjectsController < Projects::ApplicationController
def
whitelist_query_limiting
def
whitelist_query_limiting
Gitlab
::
QueryLimiting
.
whitelist
(
'https://gitlab.com/gitlab-org/gitlab-ce/issues/42440'
)
Gitlab
::
QueryLimiting
.
whitelist
(
'https://gitlab.com/gitlab-org/gitlab-ce/issues/42440'
)
end
end
def
export_project_path
@export_project_path
||=
@project
.
export_project_path
end
def
export_project_object_storage?
@project
.
export_project_object_exists?
end
end
end
app/models/import_export_upload.rb
0 → 100644
View file @
a2bf1641
class
ImportExportUpload
<
ActiveRecord
::
Base
include
WithUploads
include
ObjectStorage
::
BackgroundMove
belongs_to
:project
mount_uploader
:import_file
,
ImportExportUploader
mount_uploader
:export_file
,
ImportExportUploader
def
retrieve_upload
(
_identifier
,
paths
)
Upload
.
find_by
(
model:
self
,
path:
paths
)
end
end
app/models/project.rb
View file @
a2bf1641
...
@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base
...
@@ -171,6 +171,7 @@ class Project < ActiveRecord::Base
has_one
:fork_network
,
through: :fork_network_member
has_one
:fork_network
,
through: :fork_network_member
has_one
:import_state
,
autosave:
true
,
class_name:
'ProjectImportState'
,
inverse_of: :project
has_one
:import_state
,
autosave:
true
,
class_name:
'ProjectImportState'
,
inverse_of: :project
has_one
:import_export_upload
,
dependent: :destroy
# rubocop:disable Cop/ActiveRecordDependent
# Merge Requests for target project should be removed with it
# Merge Requests for target project should be removed with it
has_many
:merge_requests
,
foreign_key:
'target_project_id'
has_many
:merge_requests
,
foreign_key:
'target_project_id'
...
@@ -1712,7 +1713,7 @@ class Project < ActiveRecord::Base
...
@@ -1712,7 +1713,7 @@ class Project < ActiveRecord::Base
:started
:started
elsif
after_export_in_progress?
elsif
after_export_in_progress?
:after_export_action
:after_export_action
elsif
export_project_path
elsif
export_project_path
||
export_project_object_exists?
:finished
:finished
else
else
:none
:none
...
@@ -1727,16 +1728,21 @@ class Project < ActiveRecord::Base
...
@@ -1727,16 +1728,21 @@ class Project < ActiveRecord::Base
import_export_shared
.
after_export_in_progress?
import_export_shared
.
after_export_in_progress?
end
end
def
remove_exports
def
remove_exports
(
path
=
export_path
)
return
nil
unless
export_path
.
present?
if
path
.
present?
FileUtils
.
rm_rf
(
path
)
FileUtils
.
rm_rf
(
export_path
)
elsif
export_project_object_exists?
import_export_upload
.
remove_export_file!
import_export_upload
.
save
end
end
end
def
remove_exported_project_file
def
remove_exported_project_file
return
unless
export_project_path
.
present?
remove_exports
(
export_project_path
)
end
FileUtils
.
rm_f
(
export_project_path
)
def
export_project_object_exists?
Gitlab
::
ImportExport
.
object_storage?
&&
import_export_upload
&
.
export_file
&
.
file
end
end
def
full_path_slug
def
full_path_slug
...
...
app/services/import_export_clean_up_service.rb
View file @
a2bf1641
...
@@ -10,7 +10,9 @@ class ImportExportCleanUpService
...
@@ -10,7 +10,9 @@ class ImportExportCleanUpService
def
execute
def
execute
Gitlab
::
Metrics
.
measure
(
:import_export_clean_up
)
do
Gitlab
::
Metrics
.
measure
(
:import_export_clean_up
)
do
next
unless
File
.
directory?
(
path
)
clean_up_export_object_files
break
unless
File
.
directory?
(
path
)
clean_up_export_files
clean_up_export_files
end
end
...
@@ -21,4 +23,11 @@ class ImportExportCleanUpService
...
@@ -21,4 +23,11 @@ class ImportExportCleanUpService
def
clean_up_export_files
def
clean_up_export_files
Gitlab
::
Popen
.
popen
(
%W(find
#{
path
}
-not -path
#{
path
}
-mmin +
#{
mmin
}
-delete)
)
Gitlab
::
Popen
.
popen
(
%W(find
#{
path
}
-not -path
#{
path
}
-mmin +
#{
mmin
}
-delete)
)
end
end
def
clean_up_export_object_files
ImportExportUpload
.
where
(
'updated_at < ?'
,
mmin
.
minutes
.
ago
).
each
do
|
upload
|
upload
.
remove_export_file!
upload
.
save!
end
end
end
end
app/uploaders/import_export_uploader.rb
0 → 100644
View file @
a2bf1641
class
ImportExportUploader
<
AttachmentUploader
EXTENSION_WHITELIST
=
%w[tar.gz]
.
freeze
def
extension_whitelist
EXTENSION_WHITELIST
end
def
move_to_store
true
end
def
move_to_cache
false
end
end
app/views/projects/_export.html.haml
View file @
a2bf1641
...
@@ -31,7 +31,7 @@
...
@@ -31,7 +31,7 @@
%li
Any encrypted tokens
%li
Any encrypted tokens
%p
%p
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
-
if
project
.
export_
project_path
-
if
project
.
export_
status
==
:finished
=
link_to
'Download export'
,
download_export_project_path
(
project
),
=
link_to
'Download export'
,
download_export_project_path
(
project
),
rel:
'nofollow'
,
download:
''
,
method: :get
,
class:
"btn btn-default"
rel:
'nofollow'
,
download:
''
,
method: :get
,
class:
"btn btn-default"
=
link_to
'Generate new export'
,
generate_new_export_project_path
(
project
),
=
link_to
'Generate new export'
,
generate_new_export_project_path
(
project
),
...
...
changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml
0 → 100644
View file @
a2bf1641
---
title
:
Add Object Storage to project export
merge_request
:
20105
author
:
type
:
added
db/migrate/20180625113853_create_import_export_uploads.rb
0 → 100644
View file @
a2bf1641
class
CreateImportExportUploads
<
ActiveRecord
::
Migration
DOWNTIME
=
false
def
change
create_table
:import_export_uploads
do
|
t
|
t
.
datetime_with_timezone
:updated_at
,
null:
false
t
.
references
:project
,
index:
true
,
foreign_key:
{
on_delete: :cascade
},
unique:
true
t
.
text
:import_file
t
.
text
:export_file
end
add_index
:import_export_uploads
,
:updated_at
end
end
db/schema.rb
View file @
a2bf1641
...
@@ -949,6 +949,16 @@ ActiveRecord::Schema.define(version: 20180702120647) do
...
@@ -949,6 +949,16 @@ ActiveRecord::Schema.define(version: 20180702120647) do
add_index
"identities"
,
[
"user_id"
],
name:
"index_identities_on_user_id"
,
using: :btree
add_index
"identities"
,
[
"user_id"
],
name:
"index_identities_on_user_id"
,
using: :btree
create_table
"import_export_uploads"
,
force: :cascade
do
|
t
|
t
.
datetime_with_timezone
"updated_at"
,
null:
false
t
.
integer
"project_id"
t
.
text
"import_file"
t
.
text
"export_file"
end
add_index
"import_export_uploads"
,
[
"project_id"
],
name:
"index_import_export_uploads_on_project_id"
,
using: :btree
add_index
"import_export_uploads"
,
[
"updated_at"
],
name:
"index_import_export_uploads_on_updated_at"
,
using: :btree
create_table
"internal_ids"
,
id: :bigserial
,
force: :cascade
do
|
t
|
create_table
"internal_ids"
,
id: :bigserial
,
force: :cascade
do
|
t
|
t
.
integer
"project_id"
t
.
integer
"project_id"
t
.
integer
"usage"
,
null:
false
t
.
integer
"usage"
,
null:
false
...
...
doc/administration/raketasks/project_import_export.md
View file @
a2bf1641
...
@@ -30,5 +30,12 @@ sudo gitlab-rake gitlab:import_export:data
...
@@ -30,5 +30,12 @@ sudo gitlab-rake gitlab:import_export:data
bundle
exec
rake gitlab:import_export:data
RAILS_ENV
=
production
bundle
exec
rake gitlab:import_export:data
RAILS_ENV
=
production
```
```
In order to enable Object Storage on the Export, you can use the
[
feature flag
][
feature-flags
]
:
```
import_export_object_storage
```
[
ce-3050
]:
https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[
ce-3050
]:
https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[
feature-flags
]:
https://docs.gitlab.com/ee/api/features.html
[
tmp
]:
../../development/shared_files.md
[
tmp
]:
../../development/shared_files.md
lib/api/project_export.rb
View file @
a2bf1641
...
@@ -23,9 +23,13 @@ module API
...
@@ -23,9 +23,13 @@ module API
get
':id/export/download'
do
get
':id/export/download'
do
path
=
user_project
.
export_project_path
path
=
user_project
.
export_project_path
render_api_error!
(
'404 Not found or has expired'
,
404
)
unless
path
if
path
present_disk_file!
(
path
,
File
.
basename
(
path
),
'application/gzip'
)
present_disk_file!
(
path
,
File
.
basename
(
path
),
'application/gzip'
)
elsif
user_project
.
export_project_object_exists?
present_carrierwave_file!
(
user_project
.
import_export_upload
.
export_file
)
else
render_api_error!
(
'404 Not found or has expired'
,
404
)
end
end
end
desc
'Start export'
do
desc
'Start export'
do
...
...
lib/gitlab/import_export.rb
View file @
a2bf1641
...
@@ -40,6 +40,10 @@ module Gitlab
...
@@ -40,6 +40,10 @@ module Gitlab
"
#{
basename
[
0
..
FILENAME_LIMIT
]
}
_export.tar.gz"
"
#{
basename
[
0
..
FILENAME_LIMIT
]
}
_export.tar.gz"
end
end
def
object_storage?
Feature
.
enabled?
(
:import_export_object_storage
)
end
def
version
def
version
VERSION
VERSION
end
end
...
...
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
View file @
a2bf1641
...
@@ -24,9 +24,10 @@ module Gitlab
...
@@ -24,9 +24,10 @@ module Gitlab
end
end
def
execute
(
current_user
,
project
)
def
execute
(
current_user
,
project
)
return
unless
project
&
.
export_project_path
@project
=
project
@project
=
project
return
unless
@project
.
export_status
==
:finished
@current_user
=
current_user
@current_user
=
current_user
if
invalid?
if
invalid?
...
@@ -51,9 +52,12 @@ module Gitlab
...
@@ -51,9 +52,12 @@ module Gitlab
end
end
def
self
.
lock_file_path
(
project
)
def
self
.
lock_file_path
(
project
)
return
unless
project
&
.
export_path
return
unless
project
.
export_path
||
object_storage?
File
.
join
(
project
.
export_path
,
AFTER_EXPORT_LOCK_FILE_NAME
)
lock_path
=
project
.
import_export_shared
.
archive_path
FileUtils
.
mkdir_p
(
lock_path
)
File
.
join
(
lock_path
,
AFTER_EXPORT_LOCK_FILE_NAME
)
end
end
protected
protected
...
@@ -77,6 +81,10 @@ module Gitlab
...
@@ -77,6 +81,10 @@ module Gitlab
def
log_validation_errors
def
log_validation_errors
errors
.
full_messages
.
each
{
|
msg
|
project
.
import_export_shared
.
add_error_message
(
msg
)
}
errors
.
full_messages
.
each
{
|
msg
|
project
.
import_export_shared
.
add_error_message
(
msg
)
}
end
end
def
object_storage?
project
.
export_project_object_exists?
end
end
end
end
end
end
end
...
...
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
View file @
a2bf1641
...
@@ -38,14 +38,20 @@ module Gitlab
...
@@ -38,14 +38,20 @@ module Gitlab
private
private
def
send_file
def
send_file
export_file
=
File
.
open
(
project
.
export_project_path
)
Gitlab
::
HTTP
.
public_send
(
http_method
.
downcase
,
url
,
send_file_options
)
# rubocop:disable GitlabSecurity/PublicSend
Gitlab
::
HTTP
.
public_send
(
http_method
.
downcase
,
url
,
send_file_options
(
export_file
))
# rubocop:disable GitlabSecurity/PublicSend
ensure
ensure
export_file
.
close
if
export_file
export_file
.
close
if
export_file
&&
!
object_storage?
end
def
export_file
if
object_storage?
project
.
import_export_upload
.
export_file
.
file
.
open
else
File
.
open
(
project
.
export_project_path
)
end
end
end
def
send_file_options
(
export_file
)
def
send_file_options
{
{
body_stream:
export_file
,
body_stream:
export_file
,
headers:
headers
headers:
headers
...
@@ -53,7 +59,15 @@ module Gitlab
...
@@ -53,7 +59,15 @@ module Gitlab
end
end
def
headers
def
headers
{
'Content-Length'
=>
File
.
size
(
project
.
export_project_path
).
to_s
}
{
'Content-Length'
=>
export_size
.
to_s
}
end
def
export_size
if
object_storage?
project
.
import_export_upload
.
export_file
.
file
.
size
else
File
.
size
(
project
.
export_project_path
)
end
end
end
end
end
end
end
...
...
lib/gitlab/import_export/saver.rb
View file @
a2bf1641
...
@@ -15,15 +15,22 @@ module Gitlab
...
@@ -15,15 +15,22 @@ module Gitlab
def
save
def
save
if
compress_and_save
if
compress_and_save
remove_export_path
remove_export_path
Rails
.
logger
.
info
(
"Saved project export
#{
archive_file
}
"
)
Rails
.
logger
.
info
(
"Saved project export
#{
archive_file
}
"
)
archive_file
save_on_object_storage
if
use_object_storage?
else
else
@shared
.
error
(
Gitlab
::
ImportExport
::
Error
.
new
(
"Unable to save
#{
archive_file
}
into
#{
@shared
.
export_path
}
"
))
@shared
.
error
(
Gitlab
::
ImportExport
::
Error
.
new
(
error_message
))
false
false
end
end
rescue
=>
e
rescue
=>
e
@shared
.
error
(
e
)
@shared
.
error
(
e
)
false
false
ensure
if
use_object_storage?
remove_archive
remove_export_path
end
end
end
private
private
...
@@ -36,9 +43,29 @@ module Gitlab
...
@@ -36,9 +43,29 @@ module Gitlab
FileUtils
.
rm_rf
(
@shared
.
export_path
)
FileUtils
.
rm_rf
(
@shared
.
export_path
)
end
end
def
remove_archive
FileUtils
.
rm_rf
(
@shared
.
archive_path
)
end
def
archive_file
def
archive_file
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project:
@project
))
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project:
@project
))
end
end
def
save_on_object_storage
upload
=
ImportExportUpload
.
find_or_initialize_by
(
project:
@project
)
File
.
open
(
archive_file
)
{
|
file
|
upload
.
export_file
=
file
}
upload
.
save!
end
def
use_object_storage?
Gitlab
::
ImportExport
.
object_storage?
end
def
error_message
"Unable to save
#{
archive_file
}
into
#{
@shared
.
export_path
}
. Object storage enabled:
#{
use_object_storage?
}
"
end
end
end
end
end
end
end
spec/controllers/projects_controller_spec.rb
View file @
a2bf1641
...
@@ -790,23 +790,55 @@ describe ProjectsController do
...
@@ -790,23 +790,55 @@ describe ProjectsController do
project
.
add_master
(
user
)
project
.
add_master
(
user
)
end
end
context
'when project export is enabled'
do
context
'object storage disabled'
do
it
'returns 302'
do
before
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
stub_feature_flags
(
import_export_object_storage:
false
)
end
expect
(
response
).
to
have_gitlab_http_status
(
302
)
context
'when project export is enabled'
do
it
'returns 302'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
302
)
end
end
context
'when project export is disabled'
do
before
do
stub_application_setting
(
project_export_enabled?:
false
)
end
it
'returns 404'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
404
)
end
end
end
end
end
context
'
when project export is dis
abled'
do
context
'
object storage en
abled'
do
before
do
before
do
stub_
application_setting
(
project_export_enabled?:
fals
e
)
stub_
feature_flags
(
import_export_object_storage:
tru
e
)
end
end
it
'returns 404'
do
context
'when project export is enabled'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
it
'returns 302'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
404
)
expect
(
response
).
to
have_gitlab_http_status
(
302
)
end
end
context
'when project export is disabled'
do
before
do
stub_application_setting
(
project_export_enabled?:
false
)
end
it
'returns 404'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
404
)
end
end
end
end
end
end
end
...
...
spec/factories/import_export_uploads.rb
0 → 100644
View file @
a2bf1641
FactoryBot
.
define
do
factory
:import_export_upload
do
project
{
create
(
:project
)
}
end
end
spec/factories/projects.rb
View file @
a2bf1641
...
@@ -103,6 +103,22 @@ FactoryBot.define do
...
@@ -103,6 +103,22 @@ FactoryBot.define do
end
end
trait
:with_export
do
trait
:with_export
do
before
(
:create
)
do
|
_project
,
_evaluator
|
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
:import_export_object_storage
)
{
false
}
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
'import_export_object_storage'
)
{
false
}
end
after
(
:create
)
do
|
project
,
_evaluator
|
ProjectExportWorker
.
new
.
perform
(
project
.
creator
.
id
,
project
.
id
)
end
end
trait
:with_object_export
do
before
(
:create
)
do
|
_project
,
_evaluator
|
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
:import_export_object_storage
)
{
true
}
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
'import_export_object_storage'
)
{
true
}
end
after
(
:create
)
do
|
project
,
evaluator
|
after
(
:create
)
do
|
project
,
evaluator
|
ProjectExportWorker
.
new
.
perform
(
project
.
creator
.
id
,
project
.
id
)
ProjectExportWorker
.
new
.
perform
(
project
.
creator
.
id
,
project
.
id
)
end
end
...
...
spec/features/projects/import_export/export_file_spec.rb
View file @
a2bf1641
...
@@ -25,6 +25,7 @@ describe 'Import/Export - project export integration test', :js do
...
@@ -25,6 +25,7 @@ describe 'Import/Export - project export integration test', :js do
before
do
before
do
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
end
after
do
after
do
...
...
spec/features/projects/import_export/namespace_export_file_spec.rb
View file @
a2bf1641
...
@@ -5,6 +5,7 @@ describe 'Import/Export - Namespace export file cleanup', :js do
...
@@ -5,6 +5,7 @@ describe 'Import/Export - Namespace export file cleanup', :js do
before
do
before
do
allow
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
allow
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
end
after
do
after
do
...
...
spec/fixtures/project_export.tar.gz
0 → 100644
View file @
a2bf1641
File added
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
0 → 100644
View file @
a2bf1641
require
'spec_helper'
describe
Gitlab
::
ImportExport
::
AfterExportStrategies
::
BaseAfterExportStrategy
do
let!
(
:service
)
{
described_class
.
new
}
let!
(
:project
)
{
create
(
:project
,
:with_object_export
)
}
let
(
:shared
)
{
project
.
import_export_shared
}
let!
(
:user
)
{
create
(
:user
)
}
describe
'#execute'
do
before
do
allow
(
service
).
to
receive
(
:strategy_execute
)
stub_feature_flags
(
import_export_object_storage:
true
)
end
it
'returns if project exported file is not found'
do
allow
(
project
).
to
receive
(
:export_project_object_exists?
).
and_return
(
false
)
expect
(
service
).
not_to
receive
(
:strategy_execute
)
service
.
execute
(
user
,
project
)
end
it
'creates a lock file in the export dir'
do
allow
(
service
).
to
receive
(
:delete_after_export_lock
)
service
.
execute
(
user
,
project
)
expect
(
lock_path_exist?
).
to
be_truthy
end
context
'when the method succeeds'
do
it
'removes the lock file'
do
service
.
execute
(
user
,
project
)
expect
(
lock_path_exist?
).
to
be_falsey
end
end
context
'when the method fails'
do
before
do
allow
(
service
).
to
receive
(
:strategy_execute
).
and_call_original
end
context
'when validation fails'
do
before
do
allow
(
service
).
to
receive
(
:invalid?
).
and_return
(
true
)
end
it
'does not create the lock file'
do
expect
(
service
).
not_to
receive
(
:create_or_update_after_export_lock
)
service
.
execute
(
user
,
project
)
end
it
'does not execute main logic'
do
expect
(
service
).
not_to
receive
(
:strategy_execute
)
service
.
execute
(
user
,
project
)
end
it
'logs validation errors in shared context'
do
expect
(
service
).
to
receive
(
:log_validation_errors
)
service
.
execute
(
user
,
project
)
end
end
context
'when an exception is raised'
do
it
'removes the lock'
do
expect
{
service
.
execute
(
user
,
project
)
}.
to
raise_error
(
NotImplementedError
)
expect
(
lock_path_exist?
).
to
be_falsey
end
end
end
end
describe
'#log_validation_errors'
do
it
'add the message to the shared context'
do
errors
=
%w(test_message test_message2)
allow
(
service
).
to
receive
(
:invalid?
).
and_return
(
true
)
allow
(
service
.
errors
).
to
receive
(
:full_messages
).
and_return
(
errors
)
expect
(
shared
).
to
receive
(
:add_error_message
).
twice
.
and_call_original
service
.
execute
(
user
,
project
)
expect
(
shared
.
errors
).
to
eq
errors
end
end
describe
'#to_json'
do
it
'adds the current strategy class to the serialized attributes'
do
params
=
{
param1:
1
}
result
=
params
.
merge
(
klass:
described_class
.
to_s
).
to_json
expect
(
described_class
.
new
(
params
).
to_json
).
to
eq
result
end
end
def
lock_path_exist?
File
.
exist?
(
described_class
.
lock_file_path
(
project
))
end
end
spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
View file @
a2bf1641
...
@@ -9,6 +9,7 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
...
@@ -9,6 +9,7 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
describe
'#execute'
do
describe
'#execute'
do
before
do
before
do
allow
(
service
).
to
receive
(
:strategy_execute
)
allow
(
service
).
to
receive
(
:strategy_execute
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
end
it
'returns if project exported file is not found'
do
it
'returns if project exported file is not found'
do
...
...
spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
View file @
a2bf1641
...
@@ -24,13 +24,34 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
...
@@ -24,13 +24,34 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do
end
end
describe
'#execute'
do
describe
'#execute'
do
it
'removes the exported project file after the upload'
do
context
'without object storage'
do
allow
(
strategy
).
to
receive
(
:send_file
)
before
do
allow
(
strategy
).
to
receive
(
:handle_response_error
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
it
'removes the exported project file after the upload'
do
allow
(
strategy
).
to
receive
(
:send_file
)
allow
(
strategy
).
to
receive
(
:handle_response_error
)
expect
(
project
).
to
receive
(
:remove_exported_project_file
)
strategy
.
execute
(
user
,
project
)
end
end
context
'with object storage'
do
before
do
stub_feature_flags
(
import_export_object_storage:
true
)
end
expect
(
project
).
to
receive
(
:remove_exported_project_file
)
it
'removes the exported project file after the upload'
do
allow
(
strategy
).
to
receive
(
:send_file
)
allow
(
strategy
).
to
receive
(
:handle_response_error
)
strategy
.
execute
(
user
,
project
)
expect
(
project
).
to
receive
(
:remove_exported_project_file
)
strategy
.
execute
(
user
,
project
)
end
end
end
end
end
end
end
spec/lib/gitlab/import_export/all_models.yml
View file @
a2bf1641
...
@@ -293,6 +293,7 @@ project:
...
@@ -293,6 +293,7 @@ project:
-
deploy_tokens
-
deploy_tokens
-
settings
-
settings
-
ci_cd_settings
-
ci_cd_settings
-
import_export_upload
award_emoji
:
award_emoji
:
-
awardable
-
awardable
-
user
-
user
...
...
spec/lib/gitlab/import_export/saver_spec.rb
0 → 100644
View file @
a2bf1641
require
'spec_helper'
require
'fileutils'
describe
Gitlab
::
ImportExport
::
Saver
do
let!
(
:project
)
{
create
(
:project
,
:public
,
name:
'project'
)
}
let
(
:export_path
)
{
"
#{
Dir
.
tmpdir
}
/project_tree_saver_spec"
}
let
(
:shared
)
{
project
.
import_export_shared
}
subject
{
described_class
.
new
(
project:
project
,
shared:
shared
)
}
before
do
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
FileUtils
.
mkdir_p
(
shared
.
export_path
)
FileUtils
.
touch
(
"
#{
shared
.
export_path
}
/tmp.bundle"
)
end
after
do
FileUtils
.
rm_rf
(
export_path
)
end
context
'local archive'
do
it
'saves the repo to disk'
do
stub_feature_flags
(
import_export_object_storage:
false
)
subject
.
save
expect
(
shared
.
errors
).
to
be_empty
expect
(
Dir
.
empty?
(
shared
.
archive_path
)).
to
be
false
end
end
context
'object storage'
do
it
'saves the repo using object storage'
do
stub_feature_flags
(
import_export_object_storage:
true
)
stub_uploads_object_storage
(
ImportExportUploader
)
subject
.
save
expect
(
ImportExportUpload
.
find_by
(
project:
project
).
export_file
.
url
)
.
to
match
(
%r[
\/
uploads
\/
-
\/
system
\/
import_export_upload
\/
export_file.*]
)
end
end
end
spec/models/import_export_upload_spec.rb
0 → 100644
View file @
a2bf1641
require
'spec_helper'
describe
ImportExportUpload
do
subject
{
described_class
.
new
(
project:
create
(
:project
))
}
shared_examples
'stores the Import/Export file'
do
|
method
|
it
'stores the import file'
do
subject
.
public_send
(
"
#{
method
}
="
,
fixture_file_upload
(
'spec/fixtures/project_export.tar.gz'
))
subject
.
save!
url
=
"/uploads/-/system/import_export_upload/
#{
method
}
/
#{
subject
.
id
}
/project_export.tar.gz"
expect
(
subject
.
public_send
(
method
).
url
).
to
eq
(
url
)
end
end
context
'import'
do
it_behaves_like
'stores the Import/Export file'
,
:import_file
end
context
'export'
do
it_behaves_like
'stores the Import/Export file'
,
:export_file
end
end
spec/models/project_spec.rb
View file @
a2bf1641
...
@@ -2782,6 +2782,10 @@ describe Project do
...
@@ -2782,6 +2782,10 @@ describe Project do
let
(
:legacy_project
)
{
create
(
:project
,
:legacy_storage
,
:with_export
)
}
let
(
:legacy_project
)
{
create
(
:project
,
:legacy_storage
,
:with_export
)
}
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
before
do
stub_feature_flags
(
import_export_object_storage:
false
)
end
it
'removes the exports directory for the project'
do
it
'removes the exports directory for the project'
do
expect
(
File
.
exist?
(
project
.
export_path
)).
to
be_truthy
expect
(
File
.
exist?
(
project
.
export_path
)).
to
be_truthy
...
@@ -2830,12 +2834,14 @@ describe Project do
...
@@ -2830,12 +2834,14 @@ describe Project do
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
let
(
:project
)
{
create
(
:project
,
:with_export
)
}
it
'removes the exported project file'
do
it
'removes the exported project file'
do
stub_feature_flags
(
import_export_object_storage:
false
)
exported_file
=
project
.
export_project_path
exported_file
=
project
.
export_project_path
expect
(
File
.
exist?
(
exported_file
)).
to
be_truthy
expect
(
File
.
exist?
(
exported_file
)).
to
be_truthy
allow
(
FileUtils
).
to
receive
(
:rm_f
).
and_call_original
allow
(
FileUtils
).
to
receive
(
:rm_
r
f
).
and_call_original
expect
(
FileUtils
).
to
receive
(
:rm_f
).
with
(
exported_file
).
and_call_original
expect
(
FileUtils
).
to
receive
(
:rm_
r
f
).
with
(
exported_file
).
and_call_original
project
.
remove_exported_project_file
project
.
remove_exported_project_file
...
...
spec/requests/api/project_export_spec.rb
View file @
a2bf1641
...
@@ -192,6 +192,13 @@ describe API::ProjectExport do
...
@@ -192,6 +192,13 @@ describe API::ProjectExport do
context
'when upload complete'
do
context
'when upload complete'
do
before
do
before
do
FileUtils
.
rm_rf
(
project_after_export
.
export_path
)
FileUtils
.
rm_rf
(
project_after_export
.
export_path
)
if
project_after_export
.
export_project_object_exists?
upload
=
project_after_export
.
import_export_upload
upload
.
remove_export_file!
upload
.
save
end
end
end
it_behaves_like
'404 response'
do
it_behaves_like
'404 response'
do
...
@@ -261,6 +268,22 @@ describe API::ProjectExport do
...
@@ -261,6 +268,22 @@ describe API::ProjectExport do
it_behaves_like
'get project export download not found'
it_behaves_like
'get project export download not found'
end
end
end
end
context
'when an uploader is used'
do
before
do
stub_uploads_object_storage
(
ImportExportUploader
)
[
project
,
project_finished
,
project_after_export
].
each
do
|
p
|
p
.
add_master
(
user
)
upload
=
ImportExportUpload
.
new
(
project:
p
)
upload
.
export_file
=
fixture_file_upload
(
'spec/fixtures/project_export.tar.gz'
,
"`/tar.gz"
)
upload
.
save!
end
end
it_behaves_like
'get project download by strategy'
end
end
end
describe
'POST /projects/:project_id/export'
do
describe
'POST /projects/:project_id/export'
do
...
...
spec/services/import_export_clean_up_service_spec.rb
View file @
a2bf1641
...
@@ -38,6 +38,24 @@ describe ImportExportCleanUpService do
...
@@ -38,6 +38,24 @@ describe ImportExportCleanUpService do
end
end
end
end
context
'with uploader exports'
do
it
'removes old files'
do
upload
=
create
(
:import_export_upload
,
updated_at:
2
.
days
.
ago
,
export_file:
fixture_file_upload
(
'spec/fixtures/project_export.tar.gz'
))
expect
{
service
.
execute
}.
to
change
{
upload
.
reload
.
export_file
.
file
.
nil?
}.
to
(
true
)
end
it
'does not remove new files'
do
upload
=
create
(
:import_export_upload
,
updated_at:
1
.
hour
.
ago
,
export_file:
fixture_file_upload
(
'spec/fixtures/project_export.tar.gz'
))
expect
{
service
.
execute
}.
not_to
change
{
upload
.
reload
.
export_file
.
file
.
nil?
}
end
end
def
in_directory_with_files
(
mtime
:)
def
in_directory_with_files
(
mtime
:)
Dir
.
mktmpdir
do
|
tmpdir
|
Dir
.
mktmpdir
do
|
tmpdir
|
stub_repository_downloads_path
(
tmpdir
)
stub_repository_downloads_path
(
tmpdir
)
...
...
spec/uploaders/import_export_uploader_spec.rb
0 → 100644
View file @
a2bf1641
require
'spec_helper'
describe
ImportExportUploader
do
let
(
:model
)
{
build_stubbed
(
:import_export_upload
)
}
let
(
:upload
)
{
create
(
:upload
,
model:
model
)
}
subject
{
described_class
.
new
(
model
,
:import_file
)
}
context
"object_store is REMOTE"
do
before
do
stub_uploads_object_storage
end
include_context
'with storage'
,
described_class
::
Store
::
REMOTE
it_behaves_like
'builds correct paths'
,
store_dir:
%r[import_export_upload/import_file/]
,
upload_path:
%r[import_export_upload/import_file/]
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment