remove the Upload redefinition in MigrateUploadsWorker
This commit is contained in:
Micaël Bergeron
parent
9618e49b93
commit
59d3e84f0a
2 changed files with 115 additions and 169 deletions
|
|
@ -9,85 +9,6 @@ module ObjectStorage
|
|||
|
||||
SanityCheckError = Class.new(StandardError)
|
||||
|
||||
class Upload < ActiveRecord::Base
|
||||
# Upper limit for foreground checksum processing
|
||||
CHECKSUM_THRESHOLD = 100.megabytes
|
||||
|
||||
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
|
||||
|
||||
validates :size, presence: true
|
||||
validates :path, presence: true
|
||||
validates :model, presence: true
|
||||
validates :uploader, presence: true
|
||||
|
||||
before_save :calculate_checksum!, if: :foreground_checksummable?
|
||||
after_commit :schedule_checksum, if: :checksummable?
|
||||
|
||||
scope :stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) }
|
||||
scope :stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
|
||||
|
||||
def self.hexdigest(path)
|
||||
Digest::SHA256.file(path).hexdigest
|
||||
end
|
||||
|
||||
def absolute_path
|
||||
raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
|
||||
return path unless relative_path?
|
||||
|
||||
uploader_class.absolute_path(self)
|
||||
end
|
||||
|
||||
def calculate_checksum!
|
||||
self.checksum = nil
|
||||
return unless checksummable?
|
||||
|
||||
self.checksum = self.class.hexdigest(absolute_path)
|
||||
end
|
||||
|
||||
def build_uploader(mounted_as = nil)
|
||||
uploader_class.new(model, mounted_as).tap do |uploader|
|
||||
uploader.upload = self
|
||||
uploader.retrieve_from_store!(identifier)
|
||||
end
|
||||
end
|
||||
|
||||
def exist?
|
||||
File.exist?(absolute_path)
|
||||
end
|
||||
|
||||
def local?
|
||||
return true if store.nil?
|
||||
|
||||
store == ObjectStorage::Store::LOCAL
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def checksummable?
|
||||
checksum.nil? && local? && exist?
|
||||
end
|
||||
|
||||
def foreground_checksummable?
|
||||
checksummable? && size <= CHECKSUM_THRESHOLD
|
||||
end
|
||||
|
||||
def schedule_checksum
|
||||
UploadChecksumWorker.perform_async(id)
|
||||
end
|
||||
|
||||
def relative_path?
|
||||
!path.start_with?('/')
|
||||
end
|
||||
|
||||
def identifier
|
||||
File.basename(path)
|
||||
end
|
||||
|
||||
def uploader_class
|
||||
Object.const_get(uploader)
|
||||
end
|
||||
end
|
||||
|
||||
class MigrationResult
|
||||
attr_reader :upload
|
||||
attr_accessor :error
|
||||
|
|
|
|||
|
|
@ -7,113 +7,138 @@ describe ObjectStorage::MigrateUploadsWorker, :sidekiq do
|
|||
end
|
||||
end
|
||||
|
||||
let!(:projects) { create_list(:project, 10, :with_avatar) }
|
||||
let(:uploads) { Upload.all }
|
||||
let(:model_class) { Project }
|
||||
let(:mounted_as) { :avatar }
|
||||
let(:uploads) { Upload.all }
|
||||
let(:to_store) { ObjectStorage::Store::REMOTE }
|
||||
|
||||
before do
|
||||
stub_uploads_object_storage(AvatarUploader)
|
||||
end
|
||||
|
||||
describe '.enqueue!' do
|
||||
def enqueue!
|
||||
described_class.enqueue!(uploads, Project, mounted_as, to_store)
|
||||
end
|
||||
|
||||
it 'is guarded by .sanity_check!' do
|
||||
expect(described_class).to receive(:perform_async)
|
||||
expect(described_class).to receive(:sanity_check!)
|
||||
|
||||
enqueue!
|
||||
end
|
||||
|
||||
context 'sanity_check! fails' do
|
||||
include_context 'sanity_check! fails'
|
||||
|
||||
it 'does not enqueue a job' do
|
||||
expect(described_class).not_to receive(:perform_async)
|
||||
|
||||
expect { enqueue! }.to raise_error(described_class::SanityCheckError)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '.sanity_check!' do
|
||||
shared_examples 'raises a SanityCheckError' do
|
||||
let(:mount_point) { nil }
|
||||
|
||||
it do
|
||||
expect { described_class.sanity_check!(uploads, model_class, mount_point) }
|
||||
.to raise_error(described_class::SanityCheckError)
|
||||
end
|
||||
end
|
||||
|
||||
context 'uploader types mismatch' do
|
||||
let!(:outlier) { create(:upload, uploader: 'FileUploader') }
|
||||
|
||||
include_examples 'raises a SanityCheckError'
|
||||
end
|
||||
|
||||
context 'model types mismatch' do
|
||||
let!(:outlier) { create(:upload, model_type: 'Potato') }
|
||||
|
||||
include_examples 'raises a SanityCheckError'
|
||||
end
|
||||
|
||||
context 'mount point not found' do
|
||||
include_examples 'raises a SanityCheckError' do
|
||||
let(:mount_point) { :potato }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
def perform
|
||||
described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store)
|
||||
rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures
|
||||
# swallow
|
||||
end
|
||||
|
||||
shared_examples 'outputs correctly' do |success: 0, failures: 0|
|
||||
total = success + failures
|
||||
|
||||
if success > 0
|
||||
it 'outputs the reports' do
|
||||
expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files})
|
||||
|
||||
perform
|
||||
end
|
||||
shared_examples "correctly" do
|
||||
describe '.enqueue!' do
|
||||
def enqueue!
|
||||
described_class.enqueue!(uploads, Project, mounted_as, to_store)
|
||||
end
|
||||
|
||||
if failures > 0
|
||||
it 'outputs upload failures' do
|
||||
expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/)
|
||||
it 'is guarded by .sanity_check!' do
|
||||
expect(described_class).to receive(:perform_async)
|
||||
expect(described_class).to receive(:sanity_check!)
|
||||
|
||||
perform
|
||||
enqueue!
|
||||
end
|
||||
|
||||
context 'sanity_check! fails' do
|
||||
include_context 'sanity_check! fails'
|
||||
|
||||
it 'does not enqueue a job' do
|
||||
expect(described_class).not_to receive(:perform_async)
|
||||
|
||||
expect { enqueue! }.to raise_error(described_class::SanityCheckError)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
it_behaves_like 'outputs correctly', success: 10
|
||||
describe '.sanity_check!' do
|
||||
shared_examples 'raises a SanityCheckError' do
|
||||
let(:mount_point) { nil }
|
||||
|
||||
it 'migrates files' do
|
||||
perform
|
||||
|
||||
aggregate_failures do
|
||||
projects.each do |project|
|
||||
expect(project.reload.avatar.upload.local?).to be_falsey
|
||||
it do
|
||||
expect { described_class.sanity_check!(uploads, model_class, mount_point) }
|
||||
.to raise_error(described_class::SanityCheckError)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'migration is unsuccessful' do
|
||||
before do
|
||||
allow_any_instance_of(ObjectStorage::Concern).to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.")
|
||||
stub_const("WrongModel", Class.new)
|
||||
end
|
||||
|
||||
it_behaves_like 'outputs correctly', failures: 10
|
||||
context 'uploader types mismatch' do
|
||||
let!(:outlier) { create(:upload, uploader: 'GitlabUploader') }
|
||||
|
||||
include_examples 'raises a SanityCheckError'
|
||||
end
|
||||
|
||||
context 'model types mismatch' do
|
||||
let!(:outlier) { create(:upload, model_type: 'WrongModel') }
|
||||
|
||||
include_examples 'raises a SanityCheckError'
|
||||
end
|
||||
|
||||
context 'mount point not found' do
|
||||
include_examples 'raises a SanityCheckError' do
|
||||
let(:mount_point) { :potato }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
def perform
|
||||
described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store)
|
||||
rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures
|
||||
# swallow
|
||||
end
|
||||
|
||||
shared_examples 'outputs correctly' do |success: 0, failures: 0|
|
||||
total = success + failures
|
||||
|
||||
if success > 0
|
||||
it 'outputs the reports' do
|
||||
expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files})
|
||||
|
||||
perform
|
||||
end
|
||||
end
|
||||
|
||||
if failures > 0
|
||||
it 'outputs upload failures' do
|
||||
expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/)
|
||||
|
||||
perform
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
it_behaves_like 'outputs correctly', success: 10
|
||||
|
||||
it 'migrates files' do
|
||||
perform
|
||||
|
||||
expect(Upload.where(store: ObjectStorage::Store::LOCAL).count).to eq(0)
|
||||
end
|
||||
|
||||
context 'migration is unsuccessful' do
|
||||
before do
|
||||
allow_any_instance_of(ObjectStorage::Concern)
|
||||
.to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.")
|
||||
end
|
||||
|
||||
it_behaves_like 'outputs correctly', failures: 10
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "for AvatarUploader" do
|
||||
let!(:projects) { create_list(:project, 10, :with_avatar) }
|
||||
let(:mounted_as) { :avatar }
|
||||
|
||||
before do
|
||||
stub_uploads_object_storage(AvatarUploader)
|
||||
end
|
||||
|
||||
it_behaves_like "correctly"
|
||||
end
|
||||
|
||||
context "for FileUploader" do
|
||||
let!(:projects) { create_list(:project, 10) }
|
||||
let(:secret) { SecureRandom.hex }
|
||||
let(:mounted_as) { nil }
|
||||
|
||||
before do
|
||||
stub_uploads_object_storage(FileUploader)
|
||||
|
||||
projects.map do |project|
|
||||
uploader = FileUploader.new(project)
|
||||
uploader.store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))
|
||||
end
|
||||
end
|
||||
|
||||
it_behaves_like "correctly"
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue