From 10cc6cb56f7e7f72b9366b2f7edfa866dccca8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Mon, 2 Apr 2018 15:27:01 -0400 Subject: [PATCH] backport missing object storage worker specs --- .../background_move_worker_spec.rb | 146 ++++++++++++++++++ .../migrate_uploads_worker_spec.rb | 119 ++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 spec/uploaders/workers/object_storage/background_move_worker_spec.rb create mode 100644 spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb diff --git a/spec/uploaders/workers/object_storage/background_move_worker_spec.rb b/spec/uploaders/workers/object_storage/background_move_worker_spec.rb new file mode 100644 index 00000000000..b34f427fd8a --- /dev/null +++ b/spec/uploaders/workers/object_storage/background_move_worker_spec.rb @@ -0,0 +1,146 @@ +require 'spec_helper' + +describe ObjectStorage::BackgroundMoveWorker do + let(:local) { ObjectStorage::Store::LOCAL } + let(:remote) { ObjectStorage::Store::REMOTE } + + def perform + described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id) + end + + context 'for LFS' do + let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } + let(:uploader_class) { LfsObjectUploader } + let(:subject_class) { LfsObject } + let(:file_field) { :file } + let(:subject_id) { lfs_object.id } + + context 'when object storage is enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'uploads object to storage' do + expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote) + end + + context 'when background upload is disabled' do + before do + allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false } + end + + it 'is skipped' do + expect { perform }.not_to change { lfs_object.reload.file_store } + end + end + end + + context 'when object storage is disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it "doesn't migrate files" do + perform + + expect(lfs_object.reload.file_store).to eq(local) + end + end + end + + context 'for legacy artifacts' do + let(:build) { create(:ci_build, :legacy_artifacts) } + let(:uploader_class) { LegacyArtifactUploader } + let(:subject_class) { Ci::Build } + let(:file_field) { :artifacts_file } + let(:subject_id) { build.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(build.reload.artifacts_file_store).to eq(remote) + end + + context 'for artifacts_metadata' do + let(:file_field) { :artifacts_metadata } + + it 'migrates metadata to remote storage' do + perform + + expect(build.reload.artifacts_metadata_store).to eq(remote) + end + end + end + end + end + + context 'for job artifacts' do + let(:artifact) { create(:ci_job_artifact, :archive) } + let(:uploader_class) { JobArtifactUploader } + let(:subject_class) { Ci::JobArtifact } + let(:file_field) { :file } + let(:subject_id) { artifact.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(artifact.reload.file_store).to eq(remote) + end + end + end + end + + context 'for uploads' do + let!(:project) { create(:project, :with_avatar) } + let(:uploader_class) { AvatarUploader } + let(:file_field) { :avatar } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_uploads_object_storage(uploader_class, background_upload: true) + end + + describe 'supports using the model' do + let(:subject_class) { project.class } + let(:subject_id) { project.id } + + it "migrates file to remote storage" do + perform + + expect(project.reload.avatar.file_storage?).to be_falsey + end + end + + describe 'supports using the Upload' do + let(:subject_class) { Upload } + let(:subject_id) { project.avatar.upload.id } + + it "migrates file to remote storage" do + perform + + expect(project.reload.avatar.file_storage?).to be_falsey + end + end + end + end + end +end diff --git a/spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb b/spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb new file mode 100644 index 00000000000..7a7dcb71680 --- /dev/null +++ b/spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb @@ -0,0 +1,119 @@ +require 'spec_helper' + +describe ObjectStorage::MigrateUploadsWorker, :sidekiq do + shared_context 'sanity_check! fails' do + before do + expect(described_class).to receive(:sanity_check!).and_raise(described_class::SanityCheckError) + end + end + + let!(:projects) { create_list(:project, 10, :with_avatar) } + let(:uploads) { Upload.all } + let(:model_class) { Project } + let(:mounted_as) { :avatar } + let(:to_store) { ObjectStorage::Store::REMOTE } + + before do + stub_uploads_object_storage(AvatarUploader) + end + + describe '.enqueue!' do + def enqueue! + described_class.enqueue!(uploads, Project, mounted_as, to_store) + end + + it 'is guarded by .sanity_check!' do + expect(described_class).to receive(:perform_async) + expect(described_class).to receive(:sanity_check!) + + enqueue! + end + + context 'sanity_check! fails' do + include_context 'sanity_check! fails' + + it 'does not enqueue a job' do + expect(described_class).not_to receive(:perform_async) + + expect { enqueue! }.to raise_error(described_class::SanityCheckError) + end + end + end + + describe '.sanity_check!' do + shared_examples 'raises a SanityCheckError' do + let(:mount_point) { nil } + + it do + expect { described_class.sanity_check!(uploads, model_class, mount_point) } + .to raise_error(described_class::SanityCheckError) + end + end + + context 'uploader types mismatch' do + let!(:outlier) { create(:upload, uploader: 'FileUploader') } + + include_examples 'raises a SanityCheckError' + end + + context 'model types mismatch' do + let!(:outlier) { create(:upload, model_type: 'Potato') } + + include_examples 'raises a SanityCheckError' + end + + context 'mount point not found' do + include_examples 'raises a SanityCheckError' do + let(:mount_point) { :potato } + end + end + end + + describe '#perform' do + def perform + described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store) + rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures + # swallow + end + + shared_examples 'outputs correctly' do |success: 0, failures: 0| + total = success + failures + + if success > 0 + it 'outputs the reports' do + expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files}) + + perform + end + end + + if failures > 0 + it 'outputs upload failures' do + expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/) + + perform + end + end + end + + it_behaves_like 'outputs correctly', success: 10 + + it 'migrates files' do + perform + + aggregate_failures do + projects.each do |project| + expect(project.reload.avatar.upload.local?).to be_falsey + end + end + end + + context 'migration is unsuccessful' do + before do + allow_any_instance_of(ObjectStorage::Concern).to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.") + end + + it_behaves_like 'outputs correctly', failures: 10 + end + end +end