From 3a0ad99d59506592e8d5c6abf0de0fc2104f0bf2 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:08:02 -0800 Subject: [PATCH] Add untracked files to uploads --- .../populate_untracked_uploads.rb | 144 ++++- .../populate_untracked_uploads_spec.rb | 581 ++++++++++++++++++ .../track_untracked_uploads_spec.rb | 61 +- 3 files changed, 752 insertions(+), 34 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 6dbef41cff8..acd424f4558 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,27 +4,149 @@ module Gitlab class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/note\/attachment\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Note', + }, + { + pattern: /\A-\/system\/user\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'User', + }, + { + pattern: /\A-\/system\/group\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Group', + }, + { + pattern: /\A-\/system\/project\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Project', + }, + { + pattern: FILE_UPLOADER_PATH_PATTERN, + uploader: 'FileUploader', + model_type: 'Project' + }, + ] + scope :untracked, -> { where(tracked: false) } def ensure_tracked! - # TODO - # unless unhashed_upload_file.in_uploads? - # unhashed_upload_file.add_to_uploads - # end - # - # unhashed_upload_file.mark_as_tracked + return if persisted? && tracked? + + unless in_uploads? + add_to_uploads + end + + mark_as_tracked end - def model_id - # TODO + def in_uploads? + # Even though we are checking relative paths, path is enough to + # uniquely identify uploads. There is no ambiguity between + # FileUploader paths and other Uploader paths because we use the /-/ + # separator kind of like an escape character. Project full_path will + # never conflict with an upload path starting with "uploads/-/". + Upload.exists?(path: upload_path) end - def model_type - # TODO + def add_to_uploads + Upload.create!( + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) + end + + def mark_as_tracked + self.tracked = true + self.save! + end + + def upload_path + # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(/\A\//, '') # remove leading slash + else + path_relative_to_carrierwave_root + end end def uploader - # TODO + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:uploader] + end + end + end + + def model_type + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:model_type] + end + end + end + + def model_id + PATH_PATTERNS.each do |path_pattern_map| + matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd && matchd[1] + end + + # Only the FileUploader pattern will not match an ID + file_uploader_model_id + end + + def file_size + File.size(path) + end + + # Not including a leading slash + def path_relative_to_upload_dir + @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + end + + # Not including a leading slash + def path_relative_to_carrierwave_root + "uploads/#{path_relative_to_upload_dir}" + end + + private + + def file_uploader_model_id + pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ + matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] + project = Project.find_by_full_path(full_path) + project.id.to_s end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb new file mode 100644 index 00000000000..ae6a712f2ee --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -0,0 +1,581 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + let!(:uploads) { table(:uploads) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + context 'with untracked files and tracked files in unhashed_upload_files' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + + # Tracked, by doing normal file upload + uploaded_file = fixture_file_upload(fixture) + user1.update!(avatar: uploaded_file) + project1.update!(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(logo: uploaded_file) + + # Untracked, by doing normal file upload then later deleting records from DB + uploaded_file = fixture_file_upload(fixture) + user2.update!(avatar: uploaded_file) + project2.update!(avatar: uploaded_file) + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(header_logo: uploaded_file) + + # Unhashed upload files created by PrepareUnhashedUploads + unhashed_upload_files.create!(path: appearance.logo.file.file) + unhashed_upload_files.create!(path: appearance.header_logo.file.file) + unhashed_upload_files.create!(path: user1.avatar.file.file) + unhashed_upload_files.create!(path: user2.avatar.file.file) + unhashed_upload_files.create!(path: project1.avatar.file.file) + unhashed_upload_files.create!(path: project2.avatar.file.file) + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + + user2.uploads.delete_all + project2.uploads.delete_all + appearance.uploads.last.destroy + end + + it 'adds untracked files to the uploads table' do + expect do + described_class.new.perform(1, 1000) + end.to change { uploads.count }.from(4).to(8) + + expect(user2.uploads.count).to eq(1) + expect(project2.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + + it 'does not create duplicate uploads of already tracked files' do + described_class.new.perform(1, 1000) + + expect(user1.uploads.count).to eq(1) + expect(project1.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + end + + context 'with no untracked files' do + it 'does not add to the uploads table (and does not raise error)' do + expect do + described_class.new.perform(1, 1000) + end.not_to change { uploads.count }.from(0) + end + end +end + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + + describe '#ensure_tracked!' do + let(:user1) { create(:user) } + + context 'when the file is already in the uploads table' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + + before do + upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) + end + + it 'does not add an upload' do + expect do + unhashed_upload_file.ensure_tracked! + end.to_not change { upload_class.count }.from(1) + end + end + end + + describe '#add_to_uploads' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for an appearance header_logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.header_logo.file.file) } + + before do + appearance.update!(header_logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:note) { create(:note) } + let(:unhashed_upload_file) { described_class.create!(path: note.attachment.file.file) } + + before do + note.update!(attachment: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/note/attachment/#{note.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => note.id, + "model_type" => "Note", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a user avatar file path' do + let(:user) { create(:user) } + let(:unhashed_upload_file) { described_class.create!(path: user.avatar.file.file) } + + before do + user.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/user/avatar/#{user.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => user.id, + "model_type" => "User", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a group avatar file path' do + let(:group) { create(:group) } + let(:unhashed_upload_file) { described_class.create!(path: group.avatar.file.file) } + + before do + group.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/group/avatar/#{group.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => group.id, + "model_type" => "Group", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/project/avatar/#{project.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute # Markdown upload + unhashed_upload_file.save! + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + random_hex = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "#{random_hex}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "FileUploader" + }) + end + end + end + + describe '#mark_as_tracked' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'saves the record with tracked set to true' do + expect do + unhashed_upload_file.mark_as_tracked + end.to change { unhashed_upload_file.tracked }.from(false).to(true) + + expect(unhashed_upload_file.persisted?).to be_truthy + end + end + + describe '#upload_path' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:random_hex) { SecureRandom.hex } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + + it 'returns the file path relative to the project directory in uploads' do + expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg") + end + end + end + + describe '#uploader' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns FileUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('FileUploader') + end + end + end + + describe '#model_type' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns Note as a string' do + expect(unhashed_upload_file.model_type).to eq('Note') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns User as a string' do + expect(unhashed_upload_file.model_type).to eq('User') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns Group as a string' do + expect(unhashed_upload_file.model_type).to eq('Group') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + end + + describe '#model_id' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq(project.id.to_s) + end + end + end + + describe '#file_size' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(appearance.logo.size) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.avatar.size) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size) + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 5c1113a5e47..8db41786397 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -2,6 +2,10 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') describe TrackUntrackedUploads, :migration, :sidekiq do + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + matcher :be_scheduled_migration do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| @@ -30,10 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end it 'has a path field long enough for really long paths' do - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' - end - migrate! max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 @@ -57,7 +57,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do uploaded_file = fixture_file_upload(fixture) user1.update(avatar: uploaded_file) project1.update(avatar: uploaded_file) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') appearance.update(logo: uploaded_file) # Untracked, by doing normal file upload then deleting records from DB @@ -65,48 +66,62 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user2.update(avatar: uploaded_file) user2.uploads.delete_all project2.update(avatar: uploaded_file) - UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') project2.uploads.delete_all appearance.update(header_logo: uploaded_file) appearance.uploads.last.destroy end - it 'schedules background migrations' do + it 'tracks untracked migrations' do Sidekiq::Testing.inline! do migrate! # Tracked uploads still exist - expect(user1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg", + expect(user1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg", + expect(project1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg", + expect(appearance.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project1.uploads.last.uploader).to eq('FileUploader') + expect(project1.uploads.last.attributes).to include({ + "path" => @project1_markdown_upload_path, + "uploader" => "FileUploader" + }) # Untracked uploads are now tracked - expect(user2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg", + expect(user2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg", + expect(project2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.count).to eq(2) + expect(appearance.reload.uploads.count).to eq(2) expect(appearance.uploads.last.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg", + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project2.uploads.last.uploader).to eq('FileUploader') + expect(project2.uploads.last.attributes).to include({ + "path" => @project2_markdown_upload_path, + "uploader" => "FileUploader" + }) + end + end + + it 'all UnhashedUploadFile records are marked as tracked' do + Sidekiq::Testing.inline! do + migrate! + + expect(UnhashedUploadFile.count).to eq(8) + expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count) end end end