Skip to content

G
gitlab-ce
Commit a210cb6b authored by Michael Kozono's avatar Michael Kozono
Browse files
Options

Rename table to untracked_files_for_uploads

parent 2ab3031b
Hide whitespace changes
Inline Side-by-side
Showing with 157 additions and 157 deletions
db/post_migrate/20171103140253_track_untracked_uploads.rb
View file @ a210cb6b
......@@ -7,31 +7,31 @@ class TrackUntrackedUploads < ActiveRecord::Migration
disable_ddl_transaction!
DOWNTIME = false
MIGRATION = 'PrepareUnhashedUploads'
MIGRATION = 'PrepareUntrackedUploads'
def up
unless table_exists?(:unhashed_upload_files)
create_table :unhashed_upload_files do |t|
unless table_exists?(:untracked_files_for_uploads)
create_table :untracked_files_for_uploads do |t|
t.string :path, null: false
t.boolean :tracked, default: false, null: false
t.timestamps_with_timezone null: false
end
end
unless index_exists?(:unhashed_upload_files, :path)
add_index :unhashed_upload_files, :path, unique: true
unless index_exists?(:untracked_files_for_uploads, :path)
add_index :untracked_files_for_uploads, :path, unique: true
end
unless index_exists?(:unhashed_upload_files, :tracked)
add_index :unhashed_upload_files, :tracked
unless index_exists?(:untracked_files_for_uploads, :tracked)
add_index :untracked_files_for_uploads, :tracked
end
BackgroundMigrationWorker.perform_async(MIGRATION)
end
def down
if table_exists?(:unhashed_upload_files)
drop_table :unhashed_upload_files
if table_exists?(:untracked_files_for_uploads)
drop_table :untracked_files_for_uploads
end
end
end
db/schema.rb
View file @ a210cb6b
......@@ -1719,15 +1719,15 @@ ActiveRecord::Schema.define(version: 20171124150326) do
add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree
add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree
create_table "unhashed_upload_files", force: :cascade do |t|
create_table "untracked_files_for_uploads", force: :cascade do |t|
t.string "path", null: false
t.boolean "tracked", default: false, null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
end
add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree
add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree
add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree
add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree
create_table "uploads", force: :cascade do |t|
t.integer "size", limit: 8, null: false
......
lib/gitlab/background_migration/populate_untracked_uploads.rb
View file @ a210cb6b
module Gitlab
module BackgroundMigration
class PopulateUntrackedUploads
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
class UntrackedFile < ActiveRecord::Base
self.table_name = 'untracked_files_for_uploads'
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z}
......@@ -84,7 +84,7 @@ module Gitlab
end
def upload_path
# UnhashedUploadFile#path is absolute, but Upload#path depends on uploader
# UntrackedFile#path is absolute, but Upload#path depends on uploader
if uploader == 'FileUploader'
# Path relative to project directory in uploads
matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN)
......@@ -118,7 +118,7 @@ module Gitlab
# Not including a leading slash
def path_relative_to_upload_dir
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/}
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/}
@path_relative_to_upload_dir ||= path.sub(base, '')
end
......@@ -218,10 +218,10 @@ module Gitlab
def perform(start_id, end_id)
return unless migrate?
files = UnhashedUploadFile.untracked.where(id: start_id..end_id)
files.each do |unhashed_upload_file|
files = UntrackedFile.untracked.where(id: start_id..end_id)
files.each do |untracked_file|
begin
unhashed_upload_file.ensure_tracked!
untracked_file.ensure_tracked!
rescue StandardError => e
Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}"
......@@ -235,7 +235,7 @@ module Gitlab
private
def migrate?
UnhashedUploadFile.table_exists? && Upload.table_exists?
UntrackedFile.table_exists? && Upload.table_exists?
end
end
end
......
lib/gitlab/background_migration/prepare_unhashed_uploads.rb → lib/gitlab/background_migration/prepare_untracked_uploads.rb
View file @ a210cb6b
module Gitlab
module BackgroundMigration
class PrepareUnhashedUploads
class PrepareUntrackedUploads
# For bulk_queue_background_migration_jobs_by_range
include Database::MigrationHelpers
......@@ -8,31 +8,31 @@ module Gitlab
UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
class UnhashedUploadFile < ActiveRecord::Base
class UntrackedFile < ActiveRecord::Base
include EachBatch
self.table_name = 'unhashed_upload_files'
self.table_name = 'untracked_files_for_uploads'
end
def perform
return unless migrate?
clear_unhashed_upload_file_paths
store_unhashed_upload_file_paths
clear_untracked_file_paths
store_untracked_file_paths
schedule_populate_untracked_uploads_jobs
end
private
def migrate?
UnhashedUploadFile.table_exists?
UntrackedFile.table_exists?
end
def clear_unhashed_upload_file_paths
UnhashedUploadFile.delete_all
def clear_untracked_file_paths
UntrackedFile.delete_all
end
def store_unhashed_upload_file_paths
def store_untracked_file_paths
return unless Dir.exist?(UPLOAD_DIR)
each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths|
......@@ -89,7 +89,7 @@ module Gitlab
end
def insert_file_path(file_path)
table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)'
table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)'
sql = if Gitlab::Database.postgresql?
"INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;"
......@@ -103,7 +103,7 @@ module Gitlab
end
def schedule_populate_untracked_uploads_jobs
bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION)
bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION)
end
end
end
......
spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
View file @ a210cb6b
require 'spec_helper'
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:unhashed_upload_files) { table(:unhashed_upload_files) }
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
let!(:uploads) { table(:uploads) }
let(:user1) { create(:user) }
......@@ -10,7 +10,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
let(:project2) { create(:project) }
let(:appearance) { create(:appearance) }
context 'with untracked files and tracked files in unhashed_upload_files' do
context 'with untracked files and tracked files in untracked_files_for_uploads' do
before do
fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg')
......@@ -28,15 +28,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
appearance.update!(header_logo: uploaded_file)
# Unhashed upload files created by PrepareUnhashedUploads
unhashed_upload_files.create!(path: appearance.logo.file.file)
unhashed_upload_files.create!(path: appearance.header_logo.file.file)
unhashed_upload_files.create!(path: user1.avatar.file.file)
unhashed_upload_files.create!(path: user2.avatar.file.file)
unhashed_upload_files.create!(path: project1.avatar.file.file)
unhashed_upload_files.create!(path: project2.avatar.file.file)
unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}")
unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}")
# File records created by PrepareUntrackedUploads
untracked_files_for_uploads.create!(path: appearance.logo.file.file)
untracked_files_for_uploads.create!(path: appearance.header_logo.file.file)
untracked_files_for_uploads.create!(path: user1.avatar.file.file)
untracked_files_for_uploads.create!(path: user2.avatar.file.file)
untracked_files_for_uploads.create!(path: project1.avatar.file.file)
untracked_files_for_uploads.create!(path: project2.avatar.file.file)
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}")
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}")
user2.uploads.delete_all
project2.uploads.delete_all
......@@ -56,7 +56,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
it 'sets all added or confirmed tracked files to tracked' do
expect do
described_class.new.perform(1, 1000)
end.to change { unhashed_upload_files.where(tracked: true).count }.from(0).to(8)
end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8)
end
it 'does not create duplicate uploads of already tracked files' do
......@@ -68,8 +68,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
end
it 'uses the start and end batch ids [only 1st half]' do
start_id = unhashed_upload_files.all.to_a[0].id
end_id = unhashed_upload_files.all.to_a[3].id
start_id = untracked_files_for_uploads.all.to_a[0].id
end_id = untracked_files_for_uploads.all.to_a[3].id
expect do
described_class.new.perform(start_id, end_id)
......@@ -82,12 +82,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
expect(project2.uploads.count).to eq(0)
# Only 4 have been either confirmed or added to uploads
expect(unhashed_upload_files.where(tracked: true).count).to eq(4)
expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4)
end
it 'uses the start and end batch ids [only 2nd half]' do
start_id = unhashed_upload_files.all.to_a[4].id
end_id = unhashed_upload_files.all.to_a[7].id
start_id = untracked_files_for_uploads.all.to_a[4].id
end_id = untracked_files_for_uploads.all.to_a[7].id
expect do
described_class.new.perform(start_id, end_id)
......@@ -100,7 +100,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
expect(project2.uploads.count).to eq(2)
# Only 4 have been either confirmed or added to uploads
expect(unhashed_upload_files.where(tracked: true).count).to eq(4)
expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4)
end
end
......@@ -113,7 +113,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
end
end
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
include TrackUntrackedUploadsHelpers
let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload }
......@@ -122,7 +122,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
let(:user1) { create(:user) }
context 'when the file is already in the uploads table' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") }
before do
upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234)
......@@ -130,7 +130,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'does not add an upload' do
expect do
unhashed_upload_file.ensure_tracked!
untracked_file.ensure_tracked!
end.not_to change { upload_class.count }.from(1)
end
end
......@@ -142,7 +142,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for an appearance logo file path' do
let(:model) { create(:appearance) }
let(:unhashed_upload_file) { described_class.create!(path: model.logo.file.file) }
let(:untracked_file) { described_class.create!(path: model.logo.file.file) }
before do
model.update!(logo: uploaded_file)
......@@ -151,7 +151,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include({
......@@ -163,7 +163,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for an appearance header_logo file path' do
let(:model) { create(:appearance) }
let(:unhashed_upload_file) { described_class.create!(path: model.header_logo.file.file) }
let(:untracked_file) { described_class.create!(path: model.header_logo.file.file) }
before do
model.update!(header_logo: uploaded_file)
......@@ -172,7 +172,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include({
......@@ -184,7 +184,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for a pre-Markdown Note attachment file path' do
let(:model) { create(:note) }
let(:unhashed_upload_file) { described_class.create!(path: model.attachment.file.file) }
let(:untracked_file) { described_class.create!(path: model.attachment.file.file) }
before do
model.update!(attachment: uploaded_file)
......@@ -193,7 +193,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { upload_class.count }.from(0).to(1)
expect(upload_class.first.attributes).to include({
......@@ -207,7 +207,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for a user avatar file path' do
let(:model) { create(:user) }
let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) }
let(:untracked_file) { described_class.create!(path: model.avatar.file.file) }
before do
model.update!(avatar: uploaded_file)
......@@ -216,7 +216,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include({
......@@ -228,7 +228,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for a group avatar file path' do
let(:model) { create(:group) }
let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) }
let(:untracked_file) { described_class.create!(path: model.avatar.file.file) }
before do
model.update!(avatar: uploaded_file)
......@@ -237,7 +237,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include({
......@@ -251,7 +251,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for a project avatar file path' do
let(:model) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) }
let(:untracked_file) { described_class.create!(path: model.avatar.file.file) }
before do
model.update!(avatar: uploaded_file)
......@@ -260,7 +260,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
expect(model.uploads.first.attributes).to include({
......@@ -272,20 +272,20 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:model) { create(:project) }
let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") }
let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") }
before do
UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload
unhashed_upload_file.save!
untracked_file.save!
model.reload.uploads.delete_all
end
it 'creates an Upload record' do
expect do
unhashed_upload_file.add_to_uploads
untracked_file.add_to_uploads
end.to change { model.reload.uploads.count }.from(0).to(1)
hex_secret = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1]
hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1]
expect(model.uploads.first.attributes).to include({
"path" => "#{hex_secret}/rails_sample.jpg",
"uploader" => "FileUploader"
......@@ -295,250 +295,250 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
end
describe '#mark_as_tracked' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
it 'saves the record with tracked set to true' do
expect do
unhashed_upload_file.mark_as_tracked
end.to change { unhashed_upload_file.tracked }.from(false).to(true)
untracked_file.mark_as_tracked
end.to change { untracked_file.tracked }.from(false).to(true)
expect(unhashed_upload_file.persisted?).to be_truthy
expect(untracked_file.persisted?).to be_truthy
end
end
describe '#upload_path' do
context 'for an appearance logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg')
expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg')
end
end
context 'for an appearance header_logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg')
expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg')
end
end
context 'for a pre-Markdown Note attachment file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf')
expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf')
end
end
context 'for a user avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg')
expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg')
end
end
context 'for a group avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg')
expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg')
end
end
context 'for a project avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
it 'returns the file path relative to the CarrierWave root' do
expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg')
expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project) }
let(:random_hex) { SecureRandom.hex }
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") }
it 'returns the file path relative to the project directory in uploads' do
expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg")
expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg")
end
end
end
describe '#uploader' do
context 'for an appearance logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
it 'returns AttachmentUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AttachmentUploader')
expect(untracked_file.uploader).to eq('AttachmentUploader')
end
end
context 'for an appearance header_logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
it 'returns AttachmentUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AttachmentUploader')
expect(untracked_file.uploader).to eq('AttachmentUploader')
end
end
context 'for a pre-Markdown Note attachment file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
it 'returns AttachmentUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AttachmentUploader')
expect(untracked_file.uploader).to eq('AttachmentUploader')
end
end
context 'for a user avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
it 'returns AvatarUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AvatarUploader')
expect(untracked_file.uploader).to eq('AvatarUploader')
end
end
context 'for a group avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
it 'returns AvatarUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AvatarUploader')
expect(untracked_file.uploader).to eq('AvatarUploader')
end
end
context 'for a project avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
it 'returns AvatarUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('AvatarUploader')
expect(untracked_file.uploader).to eq('AvatarUploader')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
it 'returns FileUploader as a string' do
expect(unhashed_upload_file.uploader).to eq('FileUploader')
expect(untracked_file.uploader).to eq('FileUploader')
end
end
end
describe '#model_type' do
context 'for an appearance logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
it 'returns Appearance as a string' do
expect(unhashed_upload_file.model_type).to eq('Appearance')
expect(untracked_file.model_type).to eq('Appearance')
end
end
context 'for an appearance header_logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
it 'returns Appearance as a string' do
expect(unhashed_upload_file.model_type).to eq('Appearance')
expect(untracked_file.model_type).to eq('Appearance')
end
end
context 'for a pre-Markdown Note attachment file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
it 'returns Note as a string' do
expect(unhashed_upload_file.model_type).to eq('Note')
expect(untracked_file.model_type).to eq('Note')
end
end
context 'for a user avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
it 'returns User as a string' do
expect(unhashed_upload_file.model_type).to eq('User')
expect(untracked_file.model_type).to eq('User')
end
end
context 'for a group avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
it 'returns Namespace as a string' do
expect(unhashed_upload_file.model_type).to eq('Namespace')
expect(untracked_file.model_type).to eq('Namespace')
end
end
context 'for a project avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
it 'returns Project as a string' do
expect(unhashed_upload_file.model_type).to eq('Project')
expect(untracked_file.model_type).to eq('Project')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
it 'returns Project as a string' do
expect(unhashed_upload_file.model_type).to eq('Project')
expect(untracked_file.model_type).to eq('Project')
end
end
end
describe '#model_id' do
context 'for an appearance logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1')
expect(untracked_file.model_id).to eq('1')
end
end
context 'for an appearance header_logo file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1')
expect(untracked_file.model_id).to eq('1')
end
end
context 'for a pre-Markdown Note attachment file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1234')
expect(untracked_file.model_id).to eq('1234')
end
end
context 'for a user avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1234')
expect(untracked_file.model_id).to eq('1234')
end
end
context 'for a group avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1234')
expect(untracked_file.model_id).to eq('1234')
end
end
context 'for a project avatar file path' do
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq('1234')
expect(untracked_file.model_id).to eq('1234')
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") }
it 'returns the ID as a string' do
expect(unhashed_upload_file.model_id).to eq(project.id.to_s)
expect(untracked_file.model_id).to eq(project.id.to_s)
end
end
end
......@@ -549,52 +549,52 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi
context 'for an appearance logo file path' do
let(:appearance) { create(:appearance) }
let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) }
let(:untracked_file) { described_class.create!(path: appearance.logo.file.file) }
before do
appearance.update!(logo: uploaded_file)
end
it 'returns the file size' do
expect(unhashed_upload_file.file_size).to eq(35255)
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(unhashed_upload_file.file_size).to eq(appearance.logo.size)
expect(untracked_file.file_size).to eq(appearance.logo.size)
end
end
context 'for a project avatar file path' do
let(:project) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) }
let(:untracked_file) { described_class.create!(path: project.avatar.file.file) }
before do
project.update!(avatar: uploaded_file)
end
it 'returns the file size' do
expect(unhashed_upload_file.file_size).to eq(35255)
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(unhashed_upload_file.file_size).to eq(project.avatar.size)
expect(untracked_file.file_size).to eq(project.avatar.size)
end
end
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
let(:project) { create(:project) }
let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") }
let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") }
before do
UploadService.new(project, uploaded_file, FileUploader).execute
end
it 'returns the file size' do
expect(unhashed_upload_file.file_size).to eq(35255)
expect(untracked_file.file_size).to eq(35255)
end
it 'returns the same thing that CarrierWave would return' do
expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size)
expect(untracked_file.file_size).to eq(project.uploads.first.size)
end
end
end
......
spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb → spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
View file @ a210cb6b
require 'spec_helper'
describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:unhashed_upload_files) { table(:unhashed_upload_files) }
describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
let(:user1) { create(:user) }
let(:user2) { create(:user) }
......@@ -39,20 +39,20 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidek
UploadService.new(project2, uploaded_file, FileUploader).execute
end
it 'adds unhashed files to the unhashed_upload_files table' do
it 'adds unhashed files to the untracked_files_for_uploads table' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end.to change { untracked_files_for_uploads.count }.from(0).to(5)
end
end
it 'does not add hashed files to the unhashed_upload_files table' do
it 'does not add hashed files to the untracked_files_for_uploads table' do
Sidekiq::Testing.fake! do
described_class.new.perform
hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
end
end
......@@ -66,16 +66,16 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidek
end
# E.g. from a previous failed run of this background migration
context 'when there is existing data in unhashed_upload_files' do
context 'when there is existing data in untracked_files_for_uploads' do
before do
unhashed_upload_files.create(path: '/foo/bar.jpg')
untracked_files_for_uploads.create(path: '/foo/bar.jpg')
end
it 'clears existing data before adding new data' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(1).to(5)
end.to change { untracked_files_for_uploads.count }.from(1).to(5)
end
end
end
......@@ -91,7 +91,7 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidek
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.to change { unhashed_upload_files.count }.from(0).to(5)
end.to change { untracked_files_for_uploads.count }.from(0).to(5)
end
end
end
......@@ -100,11 +100,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidek
# Very new or lightly-used installations that are running this migration
# may not have an upload directory because they have no uploads.
context 'when no files were ever uploaded' do
it 'does not add to the unhashed_upload_files table (and does not raise error)' do
it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do
Sidekiq::Testing.fake! do
expect do
described_class.new.perform
end.not_to change { unhashed_upload_files.count }.from(0)
end.not_to change { untracked_files_for_uploads.count }.from(0)
end
end
end
......
spec/migrations/track_untracked_uploads_spec.rb
View file @ a210cb6b
......@@ -4,8 +4,8 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up
describe TrackUntrackedUploads, :migration, :sidekiq do
include TrackUntrackedUploadsHelpers
class UnhashedUploadFile < ActiveRecord::Base
self.table_name = 'unhashed_upload_files'
class UntrackedFile < ActiveRecord::Base
self.table_name = 'untracked_files_for_uploads'
end
matcher :be_scheduled_migration do
......@@ -29,10 +29,10 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
end
end
it 'ensures the unhashed_upload_files table exists' do
it 'ensures the untracked_files_for_uploads table exists' do
expect do
migrate!
end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true)
end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true)
end
it 'has a path field long enough for really long paths' do
......@@ -48,7 +48,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
component # filename
].flatten.join('/')
record = UnhashedUploadFile.create!(path: long_path)
record = UntrackedFile.create!(path: long_path)
expect(record.reload.path.size).to eq(5711)
end
......@@ -132,12 +132,12 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
end
end
it 'all UnhashedUploadFile records are marked as tracked' do
it 'all UntrackedFile records are marked as tracked' do
Sidekiq::Testing.inline! do
migrate!
expect(UnhashedUploadFile.count).to eq(8)
expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count)
expect(UntrackedFile.count).to eq(8)
expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count)
end
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7