Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
e61f38d7
Commit
e61f38d7
authored
7 years ago
by
Kamil Trzcinski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix data inconsistency issue for old artifacts by moving them to a currently used path
parent
c72abcef
No related merge requests found
Changes
14
Show whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
225 additions
and
58 deletions
+225
-58
app/controllers/projects/artifacts_controller.rb
app/controllers/projects/artifacts_controller.rb
+1
-1
app/controllers/uploads_controller.rb
app/controllers/uploads_controller.rb
+1
-1
app/models/ci/build.rb
app/models/ci/build.rb
+0
-32
app/uploaders/artifact_uploader.rb
app/uploaders/artifact_uploader.rb
+13
-15
app/uploaders/gitlab_uploader.rb
app/uploaders/gitlab_uploader.rb
+10
-2
app/uploaders/records_uploads.rb
app/uploaders/records_uploads.rb
+2
-2
changelogs/unreleased/migrate-artifacts-to-a-new-path.yml
changelogs/unreleased/migrate-artifacts-to-a-new-path.yml
+4
-0
db/post_migrate/20170523083112_migrate_old_artifacts.rb
db/post_migrate/20170523083112_migrate_old_artifacts.rb
+72
-0
lib/api/jobs.rb
lib/api/jobs.rb
+1
-1
lib/api/runner.rb
lib/api/runner.rb
+1
-1
lib/api/v3/builds.rb
lib/api/v3/builds.rb
+1
-1
lib/backup/artifacts.rb
lib/backup/artifacts.rb
+1
-1
lib/ci/api/builds.rb
lib/ci/api/builds.rb
+1
-1
spec/migrations/migrate_old_artifacts_spec.rb
spec/migrations/migrate_old_artifacts_spec.rb
+117
-0
No files found.
app/controllers/projects/artifacts_controller.rb
View file @
e61f38d7
...
@@ -10,7 +10,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
...
@@ -10,7 +10,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
before_action
:set_path_and_entry
,
only:
[
:file
,
:raw
]
before_action
:set_path_and_entry
,
only:
[
:file
,
:raw
]
def
download
def
download
if
artifacts_file
.
file_storag
e?
if
artifacts_file
.
local_fil
e?
send_file
artifacts_file
.
path
,
disposition:
'attachment'
send_file
artifacts_file
.
path
,
disposition:
'attachment'
else
else
redirect_to
artifacts_file
.
url
redirect_to
artifacts_file
.
url
...
...
This diff is collapsed.
Click to expand it.
app/controllers/uploads_controller.rb
View file @
e61f38d7
...
@@ -80,7 +80,7 @@ class UploadsController < ApplicationController
...
@@ -80,7 +80,7 @@ class UploadsController < ApplicationController
else
else
@uploader
=
@model
.
send
(
upload_mount
)
@uploader
=
@model
.
send
(
upload_mount
)
redirect_to
@uploader
.
url
unless
@uploader
.
file
_storage?
redirect_to
@uploader
.
url
unless
@uploader
.
local
_storage?
end
end
@uploader
@uploader
...
...
This diff is collapsed.
Click to expand it.
app/models/ci/build.rb
View file @
e61f38d7
...
@@ -255,38 +255,6 @@ module Ci
...
@@ -255,38 +255,6 @@ module Ci
Time
.
now
-
updated_at
>
15
.
minutes
.
to_i
Time
.
now
-
updated_at
>
15
.
minutes
.
to_i
end
end
##
# Deprecated
#
# This contains a hotfix for CI build data integrity, see #4246
#
# This method is used by `ArtifactUploader` to create a store_dir.
# Warning: Uploader uses it after AND before file has been stored.
#
# This method returns old path to artifacts only if it already exists.
#
def
artifacts_path
# We need the project even if it's soft deleted, because whenever
# we're really deleting the project, we'll also delete the builds,
# and in order to delete the builds, we need to know where to find
# the artifacts, which is depending on the data of the project.
# We need to retain the project in this case.
the_project
=
project
||
unscoped_project
old
=
File
.
join
(
created_at
.
utc
.
strftime
(
'%Y_%m'
),
the_project
.
ci_id
.
to_s
,
id
.
to_s
)
old_store
=
File
.
join
(
ArtifactUploader
.
artifacts_path
,
old
)
return
old
if
the_project
.
ci_id
&&
File
.
directory?
(
old_store
)
File
.
join
(
created_at
.
utc
.
strftime
(
'%Y_%m'
),
the_project
.
id
.
to_s
,
id
.
to_s
)
end
def
valid_token?
(
token
)
def
valid_token?
(
token
)
self
.
token
&&
ActiveSupport
::
SecurityUtils
.
variable_size_secure_compare
(
token
,
self
.
token
)
self
.
token
&&
ActiveSupport
::
SecurityUtils
.
variable_size_secure_compare
(
token
,
self
.
token
)
end
end
...
...
This diff is collapsed.
Click to expand it.
app/uploaders/artifact_uploader.rb
View file @
e61f38d7
class
ArtifactUploader
<
GitlabUploader
class
ArtifactUploader
<
GitlabUploader
storage
:file
storage
:file
attr_
accessor
:build
,
:field
attr_
reader
:job
,
:field
def
self
.
artifacts_path
def
self
.
local_artifacts_store
Gitlab
.
config
.
artifacts
.
path
Gitlab
.
config
.
artifacts
.
path
end
end
def
self
.
artifacts_upload_path
def
initialize
(
job
,
field
)
File
.
join
(
self
.
artifacts_path
,
'tmp/uploads/'
)
@job
,
@field
=
job
,
field
end
end
def
s
elf
.
artifacts_cache_path
def
s
tore_dir
File
.
join
(
self
.
artifacts_path
,
'tmp/cache/'
)
default_local_path
end
end
def
initialize
(
build
,
field
)
def
cache_dir
@build
,
@field
=
build
,
field
File
.
join
(
self
.
class
.
local_artifacts_store
,
'tmp/cache'
)
end
end
def
store_dir
private
File
.
join
(
self
.
class
.
artifacts_path
,
@build
.
artifacts_path
)
end
def
cache_dir
def
default_local_path
File
.
join
(
self
.
class
.
artifacts_cache_path
,
@build
.
artifacts
_path
)
File
.
join
(
self
.
class
.
local_artifacts_store
,
default
_path
)
end
end
def
filename
def
default_path
file
.
try
(
:filename
)
File
.
join
(
job
.
created_at
.
utc
.
strftime
(
'%Y_%m'
),
job
.
project_id
.
to_s
,
job
.
id
.
to_s
)
end
end
end
end
This diff is collapsed.
Click to expand it.
app/uploaders/gitlab_uploader.rb
View file @
e61f38d7
...
@@ -9,8 +9,16 @@ class GitlabUploader < CarrierWave::Uploader::Base
...
@@ -9,8 +9,16 @@ class GitlabUploader < CarrierWave::Uploader::Base
delegate
:base_dir
,
to: :class
delegate
:base_dir
,
to: :class
def
file_storage?
def
local_file?
self
.
class
.
storage
==
CarrierWave
::
Storage
::
File
local_storage?
&&
file
&
.
is_a?
(
CarrierWave
::
SanitizedFile
)
end
def
local_storage?
storage
.
is_a?
(
CarrierWave
::
Storage
::
File
)
end
def
local_cache_storage?
cache_storage
.
is_a?
(
CarrierWave
::
Storage
::
File
)
end
end
# Reduce disk IO
# Reduce disk IO
...
...
This diff is collapsed.
Click to expand it.
app/uploaders/records_uploads.rb
View file @
e61f38d7
...
@@ -16,7 +16,7 @@ module RecordsUploads
...
@@ -16,7 +16,7 @@ module RecordsUploads
#
#
# Called `after :store`
# Called `after :store`
def
record_upload
(
_tempfile
)
def
record_upload
(
_tempfile
)
return
unless
file_storag
e?
return
unless
local_fil
e?
return
unless
file
.
exists?
return
unless
file
.
exists?
Upload
.
record
(
self
)
Upload
.
record
(
self
)
...
@@ -26,7 +26,7 @@ module RecordsUploads
...
@@ -26,7 +26,7 @@ module RecordsUploads
#
#
# Called `before :remove`
# Called `before :remove`
def
destroy_upload
(
*
args
)
def
destroy_upload
(
*
args
)
return
unless
file_storag
e?
return
unless
local_fil
e?
return
unless
file
return
unless
file
Upload
.
remove_path
(
relative_path
)
Upload
.
remove_path
(
relative_path
)
...
...
This diff is collapsed.
Click to expand it.
changelogs/unreleased/migrate-artifacts-to-a-new-path.yml
0 → 100644
View file @
e61f38d7
---
title
:
Migrate artifacts to a new path
merge_request
:
author
:
This diff is collapsed.
Click to expand it.
db/post_migrate/20170523083112_migrate_old_artifacts.rb
0 → 100644
View file @
e61f38d7
class
MigrateOldArtifacts
<
ActiveRecord
::
Migration
include
Gitlab
::
Database
::
MigrationHelpers
DOWNTIME
=
false
disable_ddl_transaction!
# This uses special heuristic to find potential candidates for data migration
# Read more about this here: https://gitlab.com/gitlab-org/gitlab-ce/issues/32036#note_30422345
def
up
builds_with_artifacts
.
find_each
do
|
build
|
build
.
migrate_artifacts!
end
end
def
down
end
private
def
builds_with_artifacts
Build
.
with_artifacts
.
joins
(
'JOIN projects ON projects.id = ci_builds.project_id'
)
.
where
(
'ci_builds.id < ?'
,
min_id
||
0
)
.
where
(
'projects.ci_id IS NOT NULL'
)
.
select
(
'id'
,
'created_at'
,
'project_id'
,
'projects.ci_id AS ci_id'
)
end
def
min_id
Build
.
joins
(
'JOIN projects ON projects.id = ci_builds.project_id'
)
.
where
(
'projects.ci_id IS NULL'
)
.
pluck
(
'min(ci_builds.id)'
)
.
first
end
class
Build
<
ActiveRecord
::
Base
self
.
table_name
=
'ci_builds'
scope
:with_artifacts
,
->
()
{
where
.
not
(
artifacts_file:
[
nil
,
''
])
}
def
migrate_artifacts!
return
unless
File
.
exists?
(
source_artifacts_path
)
return
if
File
.
exists?
(
target_artifacts_path
)
ensure_target_path
FileUtils
.
move
(
source_artifacts_path
,
target_artifacts_path
)
end
private
def
source_artifacts_path
@source_artifacts_path
||=
File
.
join
(
Gitlab
.
config
.
artifacts
.
path
,
created_at
.
utc
.
strftime
(
'%Y_%m'
),
ci_id
.
to_s
,
id
.
to_s
)
end
def
target_artifacts_path
@target_artifacts_path
||=
File
.
join
(
Gitlab
.
config
.
artifacts
.
path
,
created_at
.
utc
.
strftime
(
'%Y_%m'
),
project_id
.
to_s
,
id
.
to_s
)
end
def
ensure_target_path
directory
=
File
.
dirname
(
target_artifacts_path
)
FileUtils
.
mkdir_p
(
directory
)
unless
Dir
.
exist?
(
directory
)
end
end
end
This diff is collapsed.
Click to expand it.
lib/api/jobs.rb
View file @
e61f38d7
...
@@ -225,7 +225,7 @@ module API
...
@@ -225,7 +225,7 @@ module API
end
end
def
present_artifacts!
(
artifacts_file
)
def
present_artifacts!
(
artifacts_file
)
if
!
artifacts_file
.
file_storag
e?
if
!
artifacts_file
.
local_fil
e?
redirect_to
(
build
.
artifacts_file
.
url
)
redirect_to
(
build
.
artifacts_file
.
url
)
elsif
artifacts_file
.
exists?
elsif
artifacts_file
.
exists?
present_file!
(
artifacts_file
.
path
,
artifacts_file
.
filename
)
present_file!
(
artifacts_file
.
path
,
artifacts_file
.
filename
)
...
...
This diff is collapsed.
Click to expand it.
lib/api/runner.rb
View file @
e61f38d7
...
@@ -242,7 +242,7 @@ module API
...
@@ -242,7 +242,7 @@ module API
job
=
authenticate_job!
job
=
authenticate_job!
artifacts_file
=
job
.
artifacts_file
artifacts_file
=
job
.
artifacts_file
unless
artifacts_file
.
file_storag
e?
unless
artifacts_file
.
local_fil
e?
return
redirect_to
job
.
artifacts_file
.
url
return
redirect_to
job
.
artifacts_file
.
url
end
end
...
...
This diff is collapsed.
Click to expand it.
lib/api/v3/builds.rb
View file @
e61f38d7
...
@@ -226,7 +226,7 @@ module API
...
@@ -226,7 +226,7 @@ module API
end
end
def
present_artifacts!
(
artifacts_file
)
def
present_artifacts!
(
artifacts_file
)
if
!
artifacts_file
.
file_storag
e?
if
!
artifacts_file
.
local_fil
e?
redirect_to
(
build
.
artifacts_file
.
url
)
redirect_to
(
build
.
artifacts_file
.
url
)
elsif
artifacts_file
.
exists?
elsif
artifacts_file
.
exists?
present_file!
(
artifacts_file
.
path
,
artifacts_file
.
filename
)
present_file!
(
artifacts_file
.
path
,
artifacts_file
.
filename
)
...
...
This diff is collapsed.
Click to expand it.
lib/backup/artifacts.rb
View file @
e61f38d7
...
@@ -3,7 +3,7 @@ require 'backup/files'
...
@@ -3,7 +3,7 @@ require 'backup/files'
module
Backup
module
Backup
class
Artifacts
<
Files
class
Artifacts
<
Files
def
initialize
def
initialize
super
(
'artifacts'
,
ArtifactUploader
.
artifacts_path
)
super
(
'artifacts'
,
ArtifactUploader
.
local_artifacts_store
)
end
end
def
create_files_dir
def
create_files_dir
...
...
This diff is collapsed.
Click to expand it.
lib/ci/api/builds.rb
View file @
e61f38d7
...
@@ -187,7 +187,7 @@ module Ci
...
@@ -187,7 +187,7 @@ module Ci
build
=
authenticate_build!
build
=
authenticate_build!
artifacts_file
=
build
.
artifacts_file
artifacts_file
=
build
.
artifacts_file
unless
artifacts_file
.
file_storag
e?
unless
artifacts_file
.
local_fil
e?
return
redirect_to
build
.
artifacts_file
.
url
return
redirect_to
build
.
artifacts_file
.
url
end
end
...
...
This diff is collapsed.
Click to expand it.
spec/migrations/migrate_old_artifacts_spec.rb
0 → 100644
View file @
e61f38d7
# encoding: utf-8
require
'spec_helper'
require
Rails
.
root
.
join
(
'db'
,
'post_migrate'
,
'20170523083112_migrate_old_artifacts.rb'
)
describe
MigrateOldArtifacts
do
let
(
:migration
)
{
described_class
.
new
}
let!
(
:directory
)
{
Dir
.
mktmpdir
}
before
do
allow
(
Gitlab
.
config
.
artifacts
).
to
receive
(
:path
).
and_return
(
directory
)
end
after
do
FileUtils
.
remove_entry_secure
(
directory
)
end
context
'with migratable data'
do
let
(
:project1
)
{
create
(
:empty_project
,
ci_id:
2
)
}
let
(
:project2
)
{
create
(
:empty_project
,
ci_id:
3
)
}
let
(
:project3
)
{
create
(
:empty_project
)
}
let
(
:pipeline1
)
{
create
(
:ci_empty_pipeline
,
project:
project1
)
}
let
(
:pipeline2
)
{
create
(
:ci_empty_pipeline
,
project:
project2
)
}
let
(
:pipeline3
)
{
create
(
:ci_empty_pipeline
,
project:
project3
)
}
let!
(
:build_with_legacy_artifacts
)
{
create
(
:ci_build
,
pipeline:
pipeline1
)
}
let!
(
:build_without_artifacts
)
{
create
(
:ci_build
,
pipeline:
pipeline1
)
}
let!
(
:build2
)
{
create
(
:ci_build
,
:artifacts
,
pipeline:
pipeline2
)
}
let!
(
:build3
)
{
create
(
:ci_build
,
:artifacts
,
pipeline:
pipeline3
)
}
before
do
store_artifacts_in_legacy_path
(
build_with_legacy_artifacts
)
end
it
"legacy artifacts are not accessible"
do
expect
(
build_with_legacy_artifacts
.
artifacts?
).
to
be_falsey
end
it
"legacy artifacts are set"
do
expect
(
build_with_legacy_artifacts
.
artifacts_file_identifier
).
not_to
be_nil
end
describe
'#min_id'
do
subject
{
migration
.
send
(
:min_id
)
}
it
'returns the newest build for which ci_id is not defined'
do
is_expected
.
to
eq
(
build3
.
id
)
end
end
describe
'#builds_with_artifacts'
do
subject
{
migration
.
send
(
:builds_with_artifacts
).
map
(
&
:id
)
}
it
'returns a list of builds that has artifacts and could be migrated'
do
is_expected
.
to
contain_exactly
(
build_with_legacy_artifacts
.
id
,
build2
.
id
)
end
end
describe
'#up'
do
context
'when migrating artifacts'
do
before
do
migration
.
up
end
it
'all files do have artifacts'
do
Ci
::
Build
.
with_artifacts
do
|
build
|
expect
(
build
).
to
have_artifacts
end
end
it
'artifacts are no longer present on legacy path'
do
expect
(
File
.
exist?
(
legacy_path
(
build_with_legacy_artifacts
))).
to
eq
(
false
)
end
end
context
'when there are aritfacts in old and new directory'
do
before
do
store_artifacts_in_legacy_path
(
build2
)
migration
.
up
end
it
'does not move old files'
do
expect
(
File
.
exist?
(
legacy_path
(
build2
))).
to
eq
(
true
)
end
end
end
private
def
store_artifacts_in_legacy_path
(
build
)
FileUtils
.
mkdir_p
(
legacy_path
(
build
))
FileUtils
.
copy
(
Rails
.
root
.
join
(
'spec/fixtures/ci_build_artifacts.zip'
),
File
.
join
(
legacy_path
(
build
),
"ci_build_artifacts.zip"
)
)
FileUtils
.
copy
(
Rails
.
root
.
join
(
'spec/fixtures/ci_build_artifacts_metadata.gz'
),
File
.
join
(
legacy_path
(
build
),
"ci_build_artifacts_metadata.gz"
)
)
build
.
update_columns
(
artifacts_file:
'ci_build_artifacts.zip'
,
artifacts_metadata:
'ci_build_artifacts_metadata.gz'
,
)
build
.
reload
end
def
legacy_path
(
build
)
File
.
join
(
directory
,
build
.
created_at
.
utc
.
strftime
(
'%Y_%m'
),
build
.
project
.
ci_id
.
to_s
,
build
.
id
.
to_s
)
end
end
end
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment