Process normal paths in batch containing bad paths

This commit is contained in:
Michael Kozono 2017-12-05 12:26:20 -08:00
parent 77be7efcf5
commit 869d08b581
2 changed files with 45 additions and 4 deletions

View File

@ -57,7 +57,7 @@ module Gitlab
].freeze
def to_h
{
@upload_hash ||= {
path: upload_path,
uploader: uploader,
model_type: model_type,
@ -156,8 +156,8 @@ module Gitlab
return unless migrate?
files = UntrackedFile.where(id: start_id..end_id)
insert_uploads_if_needed(files)
files.delete_all
processed_files = insert_uploads_if_needed(files)
processed_files.delete_all
drop_temp_table_if_finished
end
@ -169,9 +169,30 @@ module Gitlab
end
def insert_uploads_if_needed(files)
filtered_files = filter_existing_uploads(files)
filtered_files, error_files = filter_error_files(files)
filtered_files = filter_existing_uploads(filtered_files)
filtered_files = filter_deleted_models(filtered_files)
insert(filtered_files)
processed_files = files.where.not(id: error_files.map(&:id))
processed_files
end
def filter_error_files(files)
files.partition do |file|
begin
file.to_h
true
rescue => e
msg = <<~MSG
Error parsing path "#{file.path}":
#{e.message}
#{e.backtrace.join("\n ")}
MSG
Rails.logger.error(msg)
false
end
end
end
def filter_existing_uploads(files)

View File

@ -119,6 +119,26 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid
expect(table_exists?(:untracked_files_for_uploads)).to be_falsey
end
it 'does not block a whole batch because of one bad path' do
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a")
expect(untracked_files_for_uploads.count).to eq(9)
expect(uploads.count).to eq(4)
subject.perform(1, untracked_files_for_uploads.last.id)
expect(untracked_files_for_uploads.count).to eq(1)
expect(uploads.count).to eq(8)
end
it 'an unparseable path is shown in error output' do
bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a"
untracked_files_for_uploads.create!(path: bad_path)
expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)
subject.perform(1, untracked_files_for_uploads.last.id)
end
end
context 'with no untracked files' do