Skip to content

G
gitlab-ce
Commit a5b7f273 authored by Stan Hu's avatar Stan Hu
Browse files
Options

Merge branch 'mk-improve-background-migration-specs' into 'master' 

Improve background migration specs

See merge request gitlab-org/gitlab-ce!17162
parents 4bc17b01 0c357ac8
Show whitespace changes
Inline Side-by-side
Showing with 706 additions and 518 deletions
lib/gitlab/background_migration/populate_untracked_uploads.rb
View file @ a5b7f273
...@@ -5,157 +5,10 @@ module Gitlab ...@@ -5,157 +5,10 @@ module Gitlab
# This class processes a batch of rows in `untracked_files_for_uploads` by # This class processes a batch of rows in `untracked_files_for_uploads` by
# adding each file to the `uploads` table if it does not exist. # adding each file to the `uploads` table if it does not exist.
class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength
# This class is responsible for producing the attributes necessary to
# track an uploaded file in the `uploads` table.
class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
self.table_name = 'untracked_files_for_uploads'
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/
# These regex patterns are tested against a relative path, relative to
# the upload directory.
# For convenience, if there exists a capture group in the pattern, then
# it indicates the model_id.
PATH_PATTERNS = [
{
pattern: %r{\A-/system/appearance/logo/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Appearance'
},
{
pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Appearance'
},
{
pattern: %r{\A-/system/note/attachment/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Note'
},
{
pattern: %r{\A-/system/user/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'User'
},
{
pattern: %r{\A-/system/group/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'Namespace'
},
{
pattern: %r{\A-/system/project/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'Project'
},
{
pattern: FILE_UPLOADER_PATH,
uploader: 'FileUploader',
model_type: 'Project'
}
].freeze
def to_h
@upload_hash ||= {
path: upload_path,
uploader: uploader,
model_type: model_type,
model_id: model_id,
size: file_size,
checksum: checksum
}
end
def upload_path
# UntrackedFile#path is absolute, but Upload#path depends on uploader
@upload_path ||=
if uploader == 'FileUploader'
# Path relative to project directory in uploads
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH)
matchd[0].sub(%r{\A/}, '') # remove leading slash
else
path
end
end
def uploader
matching_pattern_map[:uploader]
end
def model_type
matching_pattern_map[:model_type]
end
def model_id
return @model_id if defined?(@model_id)
pattern = matching_pattern_map[:pattern]
matchd = path_relative_to_upload_dir.match(pattern)
# If something is captured (matchd[1] is not nil), it is a model_id
# Only the FileUploader pattern will not match an ID
@model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
end
def file_size
File.size(absolute_path)
end
def checksum
Digest::SHA256.file(absolute_path).hexdigest
end
private
def matching_pattern_map
@matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
path_relative_to_upload_dir.match(path_pattern_map[:pattern])
end
unless @matching_pattern_map
raise "Unknown upload path pattern \"#{path}\""
end
@matching_pattern_map
end
def file_uploader_model_id
matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE)
not_found_msg = <<~MSG
Could not capture project full_path from a FileUploader path:
"#{path_relative_to_upload_dir}"
MSG
raise not_found_msg unless matchd
full_path = matchd[1]
project = Project.find_by_full_path(full_path)
return nil unless project
project.id
end
# Not including a leading slash
def path_relative_to_upload_dir
upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength
base = %r{\A#{Regexp.escape(upload_dir)}/}
@path_relative_to_upload_dir ||= path.sub(base, '')
end
def absolute_path
File.join(Gitlab.config.uploads.storage_path, path)
end
end
# This class is used to query the `uploads` table.
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
end
def perform(start_id, end_id) def perform(start_id, end_id)
return unless migrate? return unless migrate?
files = UntrackedFile.where(id: start_id..end_id) files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id)
processed_files = insert_uploads_if_needed(files) processed_files = insert_uploads_if_needed(files)
processed_files.delete_all processed_files.delete_all
...@@ -165,7 +18,8 @@ module Gitlab ...@@ -165,7 +18,8 @@ module Gitlab
private private
def migrate? def migrate?
UntrackedFile.table_exists? && Upload.table_exists? Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? &&
Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists?
end end
def insert_uploads_if_needed(files) def insert_uploads_if_needed(files)
...@@ -197,7 +51,7 @@ module Gitlab ...@@ -197,7 +51,7 @@ module Gitlab
def filter_existing_uploads(files) def filter_existing_uploads(files)
paths = files.map(&:upload_path) paths = files.map(&:upload_path)
existing_paths = Upload.where(path: paths).pluck(:path).to_set existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set
files.reject do |file| files.reject do |file|
existing_paths.include?(file.upload_path) existing_paths.include?(file.upload_path)
...@@ -229,7 +83,7 @@ module Gitlab ...@@ -229,7 +83,7 @@ module Gitlab
end end
ids.each do |model_type, model_ids| ids.each do |model_type, model_ids|
model_class = Object.const_get(model_type) model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize
found_ids = model_class.where(id: model_ids.uniq).pluck(:id) found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
deleted_ids = ids[model_type] - found_ids deleted_ids = ids[model_type] - found_ids
ids[model_type] = deleted_ids ids[model_type] = deleted_ids
...@@ -249,8 +103,8 @@ module Gitlab ...@@ -249,8 +103,8 @@ module Gitlab
end end
def drop_temp_table_if_finished def drop_temp_table_if_finished
if UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup
UntrackedFile.connection.drop_table(:untracked_files_for_uploads, Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
if_exists: true) if_exists: true)
end end
end end
......
lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb 0 → 100644
View file @ a5b7f273
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module PopulateUntrackedUploadsDependencies
# This class is responsible for producing the attributes necessary to
# track an uploaded file in the `uploads` table.
class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
self.table_name = 'untracked_files_for_uploads'
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/
# These regex patterns are tested against a relative path, relative to
# the upload directory.
# For convenience, if there exists a capture group in the pattern, then
# it indicates the model_id.
PATH_PATTERNS = [
{
pattern: %r{\A-/system/appearance/logo/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Appearance'
},
{
pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Appearance'
},
{
pattern: %r{\A-/system/note/attachment/(\d+)/},
uploader: 'AttachmentUploader',
model_type: 'Note'
},
{
pattern: %r{\A-/system/user/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'User'
},
{
pattern: %r{\A-/system/group/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'Namespace'
},
{
pattern: %r{\A-/system/project/avatar/(\d+)/},
uploader: 'AvatarUploader',
model_type: 'Project'
},
{
pattern: FILE_UPLOADER_PATH,
uploader: 'FileUploader',
model_type: 'Project'
}
].freeze
def to_h
@upload_hash ||= {
path: upload_path,
uploader: uploader,
model_type: model_type,
model_id: model_id,
size: file_size,
checksum: checksum
}
end
def upload_path
# UntrackedFile#path is absolute, but Upload#path depends on uploader
@upload_path ||=
if uploader == 'FileUploader'
# Path relative to project directory in uploads
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH)
matchd[0].sub(%r{\A/}, '') # remove leading slash
else
path
end
end
def uploader
matching_pattern_map[:uploader]
end
def model_type
matching_pattern_map[:model_type]
end
def model_id
return @model_id if defined?(@model_id)
pattern = matching_pattern_map[:pattern]
matchd = path_relative_to_upload_dir.match(pattern)
# If something is captured (matchd[1] is not nil), it is a model_id
# Only the FileUploader pattern will not match an ID
@model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
end
def file_size
File.size(absolute_path)
end
def checksum
Digest::SHA256.file(absolute_path).hexdigest
end
private
def matching_pattern_map
@matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
path_relative_to_upload_dir.match(path_pattern_map[:pattern])
end
unless @matching_pattern_map
raise "Unknown upload path pattern \"#{path}\""
end
@matching_pattern_map
end
def file_uploader_model_id
matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE)
not_found_msg = <<~MSG
Could not capture project full_path from a FileUploader path:
"#{path_relative_to_upload_dir}"
MSG
raise not_found_msg unless matchd
full_path = matchd[1]
project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path)
return nil unless project
project.id
end
# Not including a leading slash
def path_relative_to_upload_dir
upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength
base = %r{\A#{Regexp.escape(upload_dir)}/}
@path_relative_to_upload_dir ||= path.sub(base, '')
end
def absolute_path
File.join(Gitlab.config.uploads.storage_path, path)
end
end
# Avoid using application code
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
end
# Avoid using application code
class Appearance < ActiveRecord::Base
self.table_name = 'appearances'
end
# Avoid using application code
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
end
# Avoid using application code
class Note < ActiveRecord::Base
self.table_name = 'notes'
end
# Avoid using application code
class User < ActiveRecord::Base
self.table_name = 'users'
end
# Since project Markdown upload paths don't contain the project ID, we have to find the
# project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons,
# the logic is somewhat complex, so I've mostly copied it in here.
class Project < ActiveRecord::Base
self.table_name = 'projects'
def self.find_by_full_path(path)
binary = Gitlab::Database.mysql? ? 'BINARY' : ''
order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)"
where_full_path_in(path).reorder(order_sql).take
end
def self.where_full_path_in(path)
cast_lower = Gitlab::Database.postgresql?
path = connection.quote(path)
where =
if cast_lower
"(LOWER(routes.path) = LOWER(#{path}))"
else
"(routes.path = #{path})"
end
joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where)
end
end
end
end
end
spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb 0 → 100644
View file @ a5b7f273
require 'spec_helper'
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
describe Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile, :migration, schema: 20180208183958 do
include MigrationsHelpers::TrackUntrackedUploadsHelpers
let!(:appearances) { table(:appearances) }
let!(:namespaces) { table(:namespaces) }
let!(:projects) { table(:projects) }
let!(:routes) { table(:routes) }
let!(:uploads) { table(:uploads) }
before(:all) do
ensure_temporary_tracking_table_exists
end
describe '#upload_path' do
def assert_upload_path(file_path, expected_upload_path)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.upload_path).to eq(expected_upload_path)
end
context 'for an appearance logo file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg')
end
end
context 'for an appearance header_logo file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf')
end
end
context 'for a user avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg')
end
end
context 'for a group avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg')
end
end
context 'for a project avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns the file path relative to the project directory in uploads' do
project = create_project
random_hex = SecureRandom.hex
assert_upload_path("/#{get_full_path(project)}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
end
end
end
describe '#uploader' do
def assert_uploader(file_path, expected_uploader)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.uploader).to eq(expected_uploader)
end
context 'for an appearance logo file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader')
end
end
context 'for an appearance header_logo file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader')
end
end
context 'for a user avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a group avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a project avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns FileUploader as a string' do
project = create_project
assert_uploader("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
end
end
end
describe '#model_type' do
def assert_model_type(file_path, expected_model_type)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.model_type).to eq(expected_model_type)
end
context 'for an appearance logo file path' do
it 'returns Appearance as a string' do
assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance')
end
end
context 'for an appearance header_logo file path' do
it 'returns Appearance as a string' do
assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns Note as a string' do
assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note')
end
end
context 'for a user avatar file path' do
it 'returns User as a string' do
assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User')
end
end
context 'for a group avatar file path' do
it 'returns Namespace as a string' do
assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace')
end
end
context 'for a project avatar file path' do
it 'returns Project as a string' do
assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns Project as a string' do
project = create_project
assert_model_type("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'Project')
end
end
end
describe '#model_id' do
def assert_model_id(file_path, expected_model_id)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.model_id).to eq(expected_model_id)
end
context 'for an appearance logo file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1)
end
end
context 'for an appearance header_logo file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1)
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234)
end
end
context 'for a user avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a group avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a project avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns the ID as a string' do
project = create_project
assert_model_id("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", project.id)
end
end
end
describe '#file_size' do
context 'for an appearance logo file path' do
let(:appearance) { create_or_update_appearance(logo: true) }
let(:untracked_file) { described_class.create!(path: get_uploads(appearance, 'Appearance').first.path) }
it 'returns the file size' do
expect(untracked_file.file_size).to eq(1062)
end
end
context 'for a project avatar file path' do
let(:project) { create_project(avatar: true) }
let(:untracked_file) { described_class.create!(path: get_uploads(project, 'Project').first.path) }
it 'returns the file size' do
expect(untracked_file.file_size).to eq(1062)
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create_project }
let(:untracked_file) { create_untracked_file("/#{get_full_path(project)}/#{get_uploads(project, 'Project').first.path}") }
before do
add_markdown_attachment(project)
end
it 'returns the file size' do
expect(untracked_file.file_size).to eq(1062)
end
end
end
def create_untracked_file(path_relative_to_upload_dir)
described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}")
end
end
spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
View file @ a5b7f273
require 'spec_helper' require 'spec_helper'
# This migration is using UploadService, which sets uploads.secret that is only # Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
# added to the DB schema in 20180129193323. Since the test isn't isolated, we describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
# just use the latest schema when testing this migration. include MigrationsHelpers::TrackUntrackedUploadsHelpers
# Ideally, the test should not use factories nor UploadService, and rely on the
# `table` helper instead.
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180129193323 do
include TrackUntrackedUploadsHelpers
subject { described_class.new } subject { described_class.new }
let!(:untracked_files_for_uploads) { described_class::UntrackedFile } let!(:appearances) { table(:appearances) }
let!(:uploads) { described_class::Upload } let!(:namespaces) { table(:namespaces) }
let!(:notes) { table(:notes) }
let!(:projects) { table(:projects) }
let!(:routes) { table(:routes) }
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
let!(:uploads) { table(:uploads) }
let!(:users) { table(:users) }
before do before do
ensure_temporary_tracking_table_exists ensure_temporary_tracking_table_exists
...@@ -19,30 +21,30 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -19,30 +21,30 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
end end
context 'with untracked files and tracked files in untracked_files_for_uploads' do context 'with untracked files and tracked files in untracked_files_for_uploads' do
let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
let!(:user1) { create(:user, :with_avatar) } let!(:user1) { create_user(avatar: true) }
let!(:user2) { create(:user, :with_avatar) } let!(:user2) { create_user(avatar: true) }
let!(:project1) { create(:project, :legacy_storage, :with_avatar) } let!(:project1) { create_project(avatar: true) }
let!(:project2) { create(:project, :legacy_storage, :with_avatar) } let!(:project2) { create_project(avatar: true) }
before do before do
UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload add_markdown_attachment(project1)
UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload add_markdown_attachment(project2)
# File records created by PrepareUntrackedUploads # File records created by PrepareUntrackedUploads
untracked_files_for_uploads.create!(path: appearance.uploads.first.path) untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').first.path)
untracked_files_for_uploads.create!(path: appearance.uploads.last.path) untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').last.path)
untracked_files_for_uploads.create!(path: user1.uploads.first.path) untracked_files_for_uploads.create!(path: get_uploads(user1, 'User').first.path)
untracked_files_for_uploads.create!(path: user2.uploads.first.path) untracked_files_for_uploads.create!(path: get_uploads(user2, 'User').first.path)
untracked_files_for_uploads.create!(path: project1.uploads.first.path) untracked_files_for_uploads.create!(path: get_uploads(project1, 'Project').first.path)
untracked_files_for_uploads.create!(path: project2.uploads.first.path) untracked_files_for_uploads.create!(path: get_uploads(project2, 'Project').first.path)
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project1).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project1, 'Project').last.path}")
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project2).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project2, 'Project').last.path}")
# Untrack 4 files # Untrack 4 files
user2.uploads.delete_all get_uploads(user2, 'User').delete_all
project2.uploads.delete_all # 2 files: avatar and a Markdown upload get_uploads(project2, 'Project').delete_all # 2 files: avatar and a Markdown upload
appearance.uploads.where("path like '%header_logo%'").delete_all get_uploads(appearance, 'Appearance').where("path like '%header_logo%'").delete_all
end end
it 'adds untracked files to the uploads table' do it 'adds untracked files to the uploads table' do
...@@ -50,9 +52,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -50,9 +52,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id) subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id)
end.to change { uploads.count }.from(4).to(8) end.to change { uploads.count }.from(4).to(8)
expect(user2.uploads.count).to eq(1) expect(get_uploads(user2, 'User').count).to eq(1)
expect(project2.uploads.count).to eq(2) expect(get_uploads(project2, 'Project').count).to eq(2)
expect(appearance.uploads.count).to eq(2) expect(get_uploads(appearance, 'Appearance').count).to eq(2)
end end
it 'deletes rows after processing them' do it 'deletes rows after processing them' do
...@@ -66,9 +68,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -66,9 +68,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
it 'does not create duplicate uploads of already tracked files' do it 'does not create duplicate uploads of already tracked files' do
subject.perform(1, untracked_files_for_uploads.last.id) subject.perform(1, untracked_files_for_uploads.last.id)
expect(user1.uploads.count).to eq(1) expect(get_uploads(user1, 'User').count).to eq(1)
expect(project1.uploads.count).to eq(2) expect(get_uploads(project1, 'Project').count).to eq(2)
expect(appearance.uploads.count).to eq(2) expect(get_uploads(appearance, 'Appearance').count).to eq(2)
end end
it 'uses the start and end batch ids [only 1st half]' do it 'uses the start and end batch ids [only 1st half]' do
...@@ -80,11 +82,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -80,11 +82,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
subject.perform(start_id, end_id) subject.perform(start_id, end_id)
end.to change { uploads.count }.from(4).to(6) end.to change { uploads.count }.from(4).to(6)
expect(user1.uploads.count).to eq(1) expect(get_uploads(user1, 'User').count).to eq(1)
expect(user2.uploads.count).to eq(1) expect(get_uploads(user2, 'User').count).to eq(1)
expect(appearance.uploads.count).to eq(2) expect(get_uploads(appearance, 'Appearance').count).to eq(2)
expect(project1.uploads.count).to eq(2) expect(get_uploads(project1, 'Project').count).to eq(2)
expect(project2.uploads.count).to eq(0) expect(get_uploads(project2, 'Project').count).to eq(0)
# Only 4 have been either confirmed or added to uploads # Only 4 have been either confirmed or added to uploads
expect(untracked_files_for_uploads.count).to eq(4) expect(untracked_files_for_uploads.count).to eq(4)
...@@ -99,11 +101,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -99,11 +101,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
subject.perform(start_id, end_id) subject.perform(start_id, end_id)
end.to change { uploads.count }.from(4).to(6) end.to change { uploads.count }.from(4).to(6)
expect(user1.uploads.count).to eq(1) expect(get_uploads(user1, 'User').count).to eq(1)
expect(user2.uploads.count).to eq(0) expect(get_uploads(user2, 'User').count).to eq(0)
expect(appearance.uploads.count).to eq(1) expect(get_uploads(appearance, 'Appearance').count).to eq(1)
expect(project1.uploads.count).to eq(2) expect(get_uploads(project1, 'Project').count).to eq(2)
expect(project2.uploads.count).to eq(2) expect(get_uploads(project2, 'Project').count).to eq(2)
# Only 4 have been either confirmed or added to uploads # Only 4 have been either confirmed or added to uploads
expect(untracked_files_for_uploads.count).to eq(4) expect(untracked_files_for_uploads.count).to eq(4)
...@@ -122,7 +124,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -122,7 +124,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
end end
it 'does not block a whole batch because of one bad path' do it 'does not block a whole batch because of one bad path' do
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a") untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a")
expect(untracked_files_for_uploads.count).to eq(9) expect(untracked_files_for_uploads.count).to eq(9)
expect(uploads.count).to eq(4) expect(uploads.count).to eq(4)
...@@ -133,7 +135,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -133,7 +135,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
end end
it 'an unparseable path is shown in error output' do it 'an unparseable path is shown in error output' do
bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a" bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a"
untracked_files_for_uploads.create!(path: bad_path) untracked_files_for_uploads.create!(path: bad_path)
expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/) expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)
...@@ -152,363 +154,100 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra ...@@ -152,363 +154,100 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
describe 'upload outcomes for each path pattern' do describe 'upload outcomes for each path pattern' do
shared_examples_for 'non_markdown_file' do shared_examples_for 'non_markdown_file' do
let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } let!(:expected_upload_attrs) { model_uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
before do before do
model.uploads.delete_all model_uploads.delete_all
end end
it 'creates an Upload record' do it 'creates an Upload record' do
expect do expect do
subject.perform(1, untracked_files_for_uploads.last.id) subject.perform(1, untracked_files_for_uploads.last.id)
end.to change { model.reload.uploads.count }.from(0).to(1) end.to change { model_uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include(expected_upload_attrs) expect(model_uploads.first.attributes).to include(expected_upload_attrs)
end end
end end
context 'for an appearance logo file path' do context 'for an appearance logo file path' do
let(:model) { create_or_update_appearance(logo: uploaded_file) } let(:model) { create_or_update_appearance(logo: true) }
let(:model_uploads) { get_uploads(model, 'Appearance') }
it_behaves_like 'non_markdown_file' it_behaves_like 'non_markdown_file'
end end
context 'for an appearance header_logo file path' do context 'for an appearance header_logo file path' do
let(:model) { create_or_update_appearance(header_logo: uploaded_file) } let(:model) { create_or_update_appearance(header_logo: true) }
let(:model_uploads) { get_uploads(model, 'Appearance') }
it_behaves_like 'non_markdown_file' it_behaves_like 'non_markdown_file'
end end
context 'for a pre-Markdown Note attachment file path' do context 'for a pre-Markdown Note attachment file path' do
let(:model) { create(:note, :with_attachment) } let(:model) { create_note(attachment: true) }
let!(:expected_upload_attrs) { Upload.where(model_type: 'Note', model_id: model.id).first.attributes.slice('path', 'uploader', 'size', 'checksum') } let!(:expected_upload_attrs) { get_uploads(model, 'Note').first.attributes.slice('path', 'uploader', 'size', 'checksum') }
let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
before do before do
Upload.where(model_type: 'Note', model_id: model.id).delete_all get_uploads(model, 'Note').delete_all
end end
# Can't use the shared example because Note doesn't have an `uploads` association # Can't use the shared example because Note doesn't have an `uploads` association
it 'creates an Upload record' do it 'creates an Upload record' do
expect do expect do
subject.perform(1, untracked_files_for_uploads.last.id) subject.perform(1, untracked_files_for_uploads.last.id)
end.to change { Upload.where(model_type: 'Note', model_id: model.id).count }.from(0).to(1) end.to change { get_uploads(model, 'Note').count }.from(0).to(1)
expect(Upload.where(model_type: 'Note', model_id: model.id).first.attributes).to include(expected_upload_attrs) expect(get_uploads(model, 'Note').first.attributes).to include(expected_upload_attrs)
end end
end end
context 'for a user avatar file path' do context 'for a user avatar file path' do
let(:model) { create(:user, :with_avatar) } let(:model) { create_user(avatar: true) }
let(:model_uploads) { get_uploads(model, 'User') }
it_behaves_like 'non_markdown_file' it_behaves_like 'non_markdown_file'
end end
context 'for a group avatar file path' do context 'for a group avatar file path' do
let(:model) { create(:group, :with_avatar) } let(:model) { create_group(avatar: true) }
let(:model_uploads) { get_uploads(model, 'Namespace') }
it_behaves_like 'non_markdown_file' it_behaves_like 'non_markdown_file'
end end
context 'for a project avatar file path' do context 'for a project avatar file path' do
let(:model) { create(:project, :legacy_storage, :with_avatar) } let(:model) { create_project(avatar: true) }
let(:model_uploads) { get_uploads(model, 'Project') }
it_behaves_like 'non_markdown_file' it_behaves_like 'non_markdown_file'
end end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:model) { create(:project, :legacy_storage) } let(:model) { create_project }
before do before do
# Upload the file # Upload the file
UploadService.new(model, uploaded_file, FileUploader).execute add_markdown_attachment(model)
# Create the untracked_files_for_uploads record # Create the untracked_files_for_uploads record
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(model)}/#{get_uploads(model, 'Project').first.path}")
# Save the expected upload attributes # Save the expected upload attributes
@expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') @expected_upload_attrs = get_uploads(model, 'Project').first.attributes.slice('path', 'uploader', 'size', 'checksum')
# Untrack the file # Untrack the file
model.reload.uploads.delete_all get_uploads(model, 'Project').delete_all
end end
it 'creates an Upload record' do it 'creates an Upload record' do
expect do expect do
subject.perform(1, untracked_files_for_uploads.last.id) subject.perform(1, untracked_files_for_uploads.last.id)
end.to change { model.reload.uploads.count }.from(0).to(1) end.to change { get_uploads(model, 'Project').count }.from(0).to(1)
expect(model.uploads.first.attributes).to include(@expected_upload_attrs) expect(get_uploads(model, 'Project').first.attributes).to include(@expected_upload_attrs)
end end
end end
end end
end end
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
include TrackUntrackedUploadsHelpers
let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload }
before(:all) do
ensure_temporary_tracking_table_exists
end
describe '#upload_path' do
def assert_upload_path(file_path, expected_upload_path)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.upload_path).to eq(expected_upload_path)
end
context 'for an appearance logo file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg')
end
end
context 'for an appearance header_logo file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf')
end
end
context 'for a user avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg')
end
end
context 'for a group avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg')
end
end
context 'for a project avatar file path' do
it 'returns the file path relative to the CarrierWave root' do
assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns the file path relative to the project directory in uploads' do
project = create(:project, :legacy_storage)
random_hex = SecureRandom.hex
assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
end
end
end
describe '#uploader' do
def assert_uploader(file_path, expected_uploader)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.uploader).to eq(expected_uploader)
end
context 'for an appearance logo file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader')
end
end
context 'for an appearance header_logo file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns AttachmentUploader as a string' do
assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader')
end
end
context 'for a user avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a group avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a project avatar file path' do
it 'returns AvatarUploader as a string' do
assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns FileUploader as a string' do
project = create(:project, :legacy_storage)
assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
end
end
end
describe '#model_type' do
def assert_model_type(file_path, expected_model_type)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.model_type).to eq(expected_model_type)
end
context 'for an appearance logo file path' do
it 'returns Appearance as a string' do
assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance')
end
end
context 'for an appearance header_logo file path' do
it 'returns Appearance as a string' do
assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance')
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns Note as a string' do
assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note')
end
end
context 'for a user avatar file path' do
it 'returns User as a string' do
assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User')
end
end
context 'for a group avatar file path' do
it 'returns Namespace as a string' do
assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace')
end
end
context 'for a project avatar file path' do
it 'returns Project as a string' do
assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns Project as a string' do
project = create(:project, :legacy_storage)
assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project')
end
end
end
describe '#model_id' do
def assert_model_id(file_path, expected_model_id)
untracked_file = create_untracked_file(file_path)
expect(untracked_file.model_id).to eq(expected_model_id)
end
context 'for an appearance logo file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1)
end
end
context 'for an appearance header_logo file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1)
end
end
context 'for a pre-Markdown Note attachment file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234)
end
end
context 'for a user avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a group avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a project avatar file path' do
it 'returns the ID as a string' do
assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234)
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
it 'returns the ID as a string' do
project = create(:project, :legacy_storage)
assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id)
end
end
end
describe '#file_size' do
context 'for an appearance logo file path' do
let(:appearance) { create_or_update_appearance(logo: uploaded_file) }
let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) }
it 'returns the file size' do
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(untracked_file.file_size).to eq(appearance.logo.size)
end
end
context 'for a project avatar file path' do
let(:project) { create(:project, :legacy_storage, avatar: uploaded_file) }
let(:untracked_file) { described_class.create!(path: project.uploads.first.path) }
it 'returns the file size' do
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(untracked_file.file_size).to eq(project.avatar.size)
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project, :legacy_storage) }
let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") }
before do
UploadService.new(project, uploaded_file, FileUploader).execute
end
it 'returns the file size' do
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(untracked_file.file_size).to eq(project.uploads.first.size)
end
end
end
def create_untracked_file(path_relative_to_upload_dir)
described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}")
end
end
spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
View file @ a5b7f273
require 'spec_helper' require 'spec_helper'
describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migration, schema: 20180129193323 do # Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
include TrackUntrackedUploadsHelpers describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
include MigrationsHelpers include MigrationsHelpers::TrackUntrackedUploadsHelpers
let!(:untracked_files_for_uploads) { described_class::UntrackedFile } let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
let!(:appearances) { table(:appearances) }
let!(:namespaces) { table(:namespaces) }
let!(:projects) { table(:projects) }
let!(:routes) { table(:routes) }
let!(:uploads) { table(:uploads) }
let!(:users) { table(:users) }
around do |example| around do |example|
# Especially important so the follow-up migration does not get run # Especially important so the follow-up migration does not get run
...@@ -15,19 +21,17 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat ...@@ -15,19 +21,17 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat
shared_examples 'prepares the untracked_files_for_uploads table' do shared_examples 'prepares the untracked_files_for_uploads table' do
context 'when files were uploaded before and after hashed storage was enabled' do context 'when files were uploaded before and after hashed storage was enabled' do
let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
let!(:user) { create(:user, :with_avatar) } let!(:user) { create_user(avatar: true) }
let!(:project1) { create(:project, :with_avatar, :legacy_storage) } let!(:project1) { create_project(avatar: true) }
let(:project2) { create(:project) } # instantiate after enabling hashed_storage let(:project2) { create_project } # instantiate after enabling hashed_storage
before do before do
# Markdown upload before enabling hashed_storage # Markdown upload before enabling hashed_storage
UploadService.new(project1, uploaded_file, FileUploader).execute add_markdown_attachment(project1)
stub_application_setting(hashed_storage_enabled: true)
# Markdown upload after enabling hashed_storage # Markdown upload after enabling hashed_storage
UploadService.new(project2, uploaded_file, FileUploader).execute add_markdown_attachment(project2, hashed_storage: true)
end end
it 'has a path field long enough for really long paths' do it 'has a path field long enough for really long paths' do
...@@ -61,7 +65,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat ...@@ -61,7 +65,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat
it 'does not add hashed files to the untracked_files_for_uploads table' do it 'does not add hashed files to the untracked_files_for_uploads table' do
described_class.new.perform described_class.new.perform
hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path hashed_file_path = get_uploads(project2, 'Project').where(uploader: 'FileUploader').first.path
expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
end end
......
spec/migrations/track_untracked_uploads_spec.rb
View file @ a5b7f273
...@@ -2,7 +2,7 @@ require 'spec_helper' ...@@ -2,7 +2,7 @@ require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads')
describe TrackUntrackedUploads, :migration, :sidekiq do describe TrackUntrackedUploads, :migration, :sidekiq do
include TrackUntrackedUploadsHelpers include MigrationsHelpers::TrackUntrackedUploadsHelpers
it 'correctly schedules the follow-up background migration' do it 'correctly schedules the follow-up background migration' do
Sidekiq::Testing.fake! do Sidekiq::Testing.fake! do
......
spec/spec_helper.rb
View file @ a5b7f273
...@@ -154,6 +154,22 @@ RSpec.configure do |config| ...@@ -154,6 +154,22 @@ RSpec.configure do |config|
Sidekiq.redis(&:flushall) Sidekiq.redis(&:flushall)
end end
# The :each scope runs "inside" the example, so this hook ensures the DB is in the
# correct state before any examples' before hooks are called. This prevents a
# problem where `ScheduleIssuesClosedAtTypeChange` (or any migration that depends
# on background migrations being run inline during test setup) can be broken by
# altering Sidekiq behavior in an unrelated spec like so:
#
# around do |example|
# Sidekiq::Testing.fake! do
# example.run
# end
# end
config.before(:context, :migration) do
schema_migrate_down!
end
# Each example may call `migrate!`, so we must ensure we are migrated down every time
config.before(:each, :migration) do config.before(:each, :migration) do
schema_migrate_down! schema_migrate_down!
end end
......
spec/support/migrations_helpers/track_untracked_uploads_helpers.rb 0 → 100644
View file @ a5b7f273
module MigrationsHelpers
module TrackUntrackedUploadsHelpers
PUBLIC_DIR = File.join(Rails.root, 'tmp', 'tests', 'public')
UPLOADS_DIR = File.join(PUBLIC_DIR, 'uploads')
SYSTEM_DIR = File.join(UPLOADS_DIR, '-', 'system')
UPLOAD_FILENAME = 'image.png'.freeze
FIXTURE_FILE_PATH = File.join(Rails.root, 'spec', 'fixtures', 'dk.png')
FIXTURE_CHECKSUM = 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'.freeze
def create_or_update_appearance(logo: false, header_logo: false)
appearance = appearances.first_or_create(title: 'foo', description: 'bar', logo: (UPLOAD_FILENAME if logo), header_logo: (UPLOAD_FILENAME if header_logo))
add_upload(appearance, 'Appearance', 'logo', 'AttachmentUploader') if logo
add_upload(appearance, 'Appearance', 'header_logo', 'AttachmentUploader') if header_logo
appearance
end
def create_group(avatar: false)
index = unique_index(:group)
group = namespaces.create(name: "group#{index}", path: "group#{index}", avatar: (UPLOAD_FILENAME if avatar))
add_upload(group, 'Group', 'avatar', 'AvatarUploader') if avatar
group
end
def create_note(attachment: false)
note = notes.create(attachment: (UPLOAD_FILENAME if attachment))
add_upload(note, 'Note', 'attachment', 'AttachmentUploader') if attachment
note
end
def create_project(avatar: false)
group = create_group
project = projects.create(namespace_id: group.id, path: "project#{unique_index(:project)}", avatar: (UPLOAD_FILENAME if avatar))
routes.create(path: "#{group.path}/#{project.path}", source_id: project.id, source_type: 'Project') # so Project.find_by_full_path works
add_upload(project, 'Project', 'avatar', 'AvatarUploader') if avatar
project
end
def create_user(avatar: false)
user = users.create(email: "foo#{unique_index(:user)}@bar.com", avatar: (UPLOAD_FILENAME if avatar), projects_limit: 100)
add_upload(user, 'User', 'avatar', 'AvatarUploader') if avatar
user
end
def unique_index(name = :unnamed)
@unique_index ||= {}
@unique_index[name] ||= 0
@unique_index[name] += 1
end
def add_upload(model, model_type, attachment_type, uploader)
file_path = upload_file_path(model, model_type, attachment_type)
path_relative_to_public = file_path.sub("#{PUBLIC_DIR}/", '')
create_file(file_path)
uploads.create!(
size: 1062,
path: path_relative_to_public,
model_id: model.id,
model_type: model_type == 'Group' ? 'Namespace' : model_type,
uploader: uploader,
checksum: FIXTURE_CHECKSUM
)
end
def add_markdown_attachment(project, hashed_storage: false)
project_dir = hashed_storage ? hashed_project_uploads_dir(project) : legacy_project_uploads_dir(project)
attachment_dir = File.join(project_dir, SecureRandom.hex)
attachment_file_path = File.join(attachment_dir, UPLOAD_FILENAME)
project_attachment_path_relative_to_project = attachment_file_path.sub("#{project_dir}/", '')
create_file(attachment_file_path)
uploads.create!(
size: 1062,
path: project_attachment_path_relative_to_project,
model_id: project.id,
model_type: 'Project',
uploader: 'FileUploader',
checksum: FIXTURE_CHECKSUM
)
end
def legacy_project_uploads_dir(project)
namespace = namespaces.find_by(id: project.namespace_id)
File.join(UPLOADS_DIR, namespace.path, project.path)
end
def hashed_project_uploads_dir(project)
File.join(UPLOADS_DIR, '@hashed', 'aa', 'aaaaaaaaaaaa')
end
def upload_file_path(model, model_type, attachment_type)
dir = File.join(upload_dir(model_type.downcase, attachment_type.to_s), model.id.to_s)
File.join(dir, UPLOAD_FILENAME)
end
def upload_dir(model_type, attachment_type)
File.join(SYSTEM_DIR, model_type, attachment_type)
end
def create_file(path)
File.delete(path) if File.exist?(path)
FileUtils.mkdir_p(File.dirname(path))
FileUtils.cp(FIXTURE_FILE_PATH, path)
end
def get_uploads(model, model_type)
uploads.where(model_type: model_type, model_id: model.id)
end
def get_full_path(project)
routes.find_by(source_id: project.id, source_type: 'Project').path
end
def ensure_temporary_tracking_table_exists
Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
end
end
end
spec/support/track_untracked_uploads_helpers.rb deleted 100644 → 0
View file @ 4bc17b01
module TrackUntrackedUploadsHelpers
def uploaded_file
fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg')
fixture_file_upload(fixture_path)
end
def ensure_temporary_tracking_table_exists
Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
end
def create_or_update_appearance(attrs)
a = Appearance.first_or_initialize(title: 'foo', description: 'bar')
a.update!(attrs)
a
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7