Commit 869d08b5 authored by Michael Kozono's avatar Michael Kozono

Process normal paths in batch containing bad paths

parent 77be7efc
...@@ -57,7 +57,7 @@ module Gitlab ...@@ -57,7 +57,7 @@ module Gitlab
].freeze ].freeze
def to_h def to_h
{ @upload_hash ||= {
path: upload_path, path: upload_path,
uploader: uploader, uploader: uploader,
model_type: model_type, model_type: model_type,
...@@ -156,8 +156,8 @@ module Gitlab ...@@ -156,8 +156,8 @@ module Gitlab
return unless migrate? return unless migrate?
files = UntrackedFile.where(id: start_id..end_id) files = UntrackedFile.where(id: start_id..end_id)
insert_uploads_if_needed(files) processed_files = insert_uploads_if_needed(files)
files.delete_all processed_files.delete_all
drop_temp_table_if_finished drop_temp_table_if_finished
end end
...@@ -169,9 +169,30 @@ module Gitlab ...@@ -169,9 +169,30 @@ module Gitlab
end end
def insert_uploads_if_needed(files) def insert_uploads_if_needed(files)
filtered_files = filter_existing_uploads(files) filtered_files, error_files = filter_error_files(files)
filtered_files = filter_existing_uploads(filtered_files)
filtered_files = filter_deleted_models(filtered_files) filtered_files = filter_deleted_models(filtered_files)
insert(filtered_files) insert(filtered_files)
processed_files = files.where.not(id: error_files.map(&:id))
processed_files
end
def filter_error_files(files)
files.partition do |file|
begin
file.to_h
true
rescue => e
msg = <<~MSG
Error parsing path "#{file.path}":
#{e.message}
#{e.backtrace.join("\n ")}
MSG
Rails.logger.error(msg)
false
end
end
end end
def filter_existing_uploads(files) def filter_existing_uploads(files)
......
...@@ -119,6 +119,26 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid ...@@ -119,6 +119,26 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
expect(table_exists?(:untracked_files_for_uploads)).to be_falsey expect(table_exists?(:untracked_files_for_uploads)).to be_falsey
end end
it 'does not block a whole batch because of one bad path' do
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a")
expect(untracked_files_for_uploads.count).to eq(9)
expect(uploads.count).to eq(4)
subject.perform(1, untracked_files_for_uploads.last.id)
expect(untracked_files_for_uploads.count).to eq(1)
expect(uploads.count).to eq(8)
end
it 'an unparseable path is shown in error output' do
bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a"
untracked_files_for_uploads.create!(path: bad_path)
expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)
subject.perform(1, untracked_files_for_uploads.last.id)
end
end end
context 'with no untracked files' do context 'with no untracked files' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment