Merge branch 'backport-os-worker-specs' into 'master'

Backport missing object storage worker specs

See merge request gitlab-org/gitlab-ce!18138

spec/uploaders/workers/object_storage/background_move_worker_spec.rb

@@ -0,0 +1,146 @@
+require 'spec_helper'
+
+describe ObjectStorage::BackgroundMoveWorker do
+  let(:local) { ObjectStorage::Store::LOCAL }
+  let(:remote) { ObjectStorage::Store::REMOTE }
+
+  def perform
+    described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id)
+  end
+
+  context 'for LFS' do
+    let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+    let(:uploader_class) { LfsObjectUploader }
+    let(:subject_class) { LfsObject }
+    let(:file_field) { :file }
+    let(:subject_id) { lfs_object.id }
+
+    context 'when object storage is enabled' do
+      before do
+        stub_lfs_object_storage(background_upload: true)
+      end
+
+      it 'uploads object to storage' do
+        expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+      end
+
+      context 'when background upload is disabled' do
+        before do
+          allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false }
+        end
+
+        it 'is skipped' do
+          expect { perform }.not_to change { lfs_object.reload.file_store }
+        end
+      end
+    end
+
+    context 'when object storage is disabled' do
+      before do
+        stub_lfs_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        perform
+
+        expect(lfs_object.reload.file_store).to eq(local)
+      end
+    end
+  end
+
+  context 'for legacy artifacts' do
+    let(:build) { create(:ci_build, :legacy_artifacts) }
+    let(:uploader_class) { LegacyArtifactUploader }
+    let(:subject_class) { Ci::Build }
+    let(:file_field) { :artifacts_file }
+    let(:subject_id) { build.id }
+
+    context 'when local storage is used' do
+      let(:store) { local }
+
+      context 'and remote storage is defined' do
+        before do
+          stub_artifacts_object_storage(background_upload: true)
+        end
+
+        it "migrates file to remote storage" do
+          perform
+
+          expect(build.reload.artifacts_file_store).to eq(remote)
+        end
+
+        context 'for artifacts_metadata' do
+          let(:file_field) { :artifacts_metadata }
+
+          it 'migrates metadata to remote storage' do
+            perform
+
+            expect(build.reload.artifacts_metadata_store).to eq(remote)
+          end
+        end
+      end
+    end
+  end
+
+  context 'for job artifacts' do
+    let(:artifact) { create(:ci_job_artifact, :archive) }
+    let(:uploader_class) { JobArtifactUploader }
+    let(:subject_class) { Ci::JobArtifact }
+    let(:file_field) { :file }
+    let(:subject_id) { artifact.id }
+
+    context 'when local storage is used' do
+      let(:store) { local }
+
+      context 'and remote storage is defined' do
+        before do
+          stub_artifacts_object_storage(background_upload: true)
+        end
+
+        it "migrates file to remote storage" do
+          perform
+
+          expect(artifact.reload.file_store).to eq(remote)
+        end
+      end
+    end
+  end
+
+  context 'for uploads' do
+    let!(:project) { create(:project, :with_avatar) }
+    let(:uploader_class) { AvatarUploader }
+    let(:file_field) { :avatar }
+
+    context 'when local storage is used' do
+      let(:store) { local }
+
+      context 'and remote storage is defined' do
+        before do
+          stub_uploads_object_storage(uploader_class, background_upload: true)
+        end
+
+        describe 'supports using the model' do
+          let(:subject_class) { project.class }
+          let(:subject_id) { project.id }
+
+          it "migrates file to remote storage" do
+            perform
+
+            expect(project.reload.avatar.file_storage?).to be_falsey
+          end
+        end
+
+        describe 'supports using the Upload' do
+          let(:subject_class) { Upload }
+          let(:subject_id) { project.avatar.upload.id }
+
+          it "migrates file to remote storage" do
+            perform
+
+            expect(project.reload.avatar.file_storage?).to be_falsey
+          end
+        end
+      end
+    end
+  end
+end

spec/uploaders/workers/object_storage/migrate_uploads_worker_spec.rb

@@ -0,0 +1,119 @@
+require 'spec_helper'
+
+describe ObjectStorage::MigrateUploadsWorker, :sidekiq do
+  shared_context 'sanity_check! fails' do
+    before do
+      expect(described_class).to receive(:sanity_check!).and_raise(described_class::SanityCheckError)
+    end
+  end
+
+  let!(:projects) { create_list(:project, 10, :with_avatar) }
+  let(:uploads) { Upload.all }
+  let(:model_class) { Project }
+  let(:mounted_as) { :avatar }
+  let(:to_store) { ObjectStorage::Store::REMOTE }
+
+  before do
+    stub_uploads_object_storage(AvatarUploader)
+  end
+
+  describe '.enqueue!' do
+    def enqueue!
+      described_class.enqueue!(uploads, Project, mounted_as, to_store)
+    end
+
+    it 'is guarded by .sanity_check!' do
+      expect(described_class).to receive(:perform_async)
+      expect(described_class).to receive(:sanity_check!)
+
+      enqueue!
+    end
+
+    context 'sanity_check! fails' do
+      include_context 'sanity_check! fails'
+
+      it 'does not enqueue a job' do
+        expect(described_class).not_to receive(:perform_async)
+
+        expect { enqueue! }.to raise_error(described_class::SanityCheckError)
+      end
+    end
+  end
+
+  describe '.sanity_check!' do
+    shared_examples 'raises a SanityCheckError' do
+      let(:mount_point) { nil }
+
+      it do
+        expect { described_class.sanity_check!(uploads, model_class, mount_point) }
+          .to raise_error(described_class::SanityCheckError)
+      end
+    end
+
+    context 'uploader types mismatch' do
+      let!(:outlier) { create(:upload, uploader: 'FileUploader') }
+
+      include_examples 'raises a SanityCheckError'
+    end
+
+    context 'model types mismatch' do
+      let!(:outlier) { create(:upload, model_type: 'Potato') }
+
+      include_examples 'raises a SanityCheckError'
+    end
+
+    context 'mount point not found' do
+      include_examples 'raises a SanityCheckError' do
+        let(:mount_point) { :potato }
+      end
+    end
+  end
+
+  describe '#perform' do
+    def perform
+      described_class.new.perform(uploads.ids, model_class.to_s, mounted_as, to_store)
+    rescue ObjectStorage::MigrateUploadsWorker::Report::MigrationFailures
+      # swallow
+    end
+
+    shared_examples 'outputs correctly' do |success: 0, failures: 0|
+      total = success + failures
+
+      if success > 0
+        it 'outputs the reports' do
+          expect(Rails.logger).to receive(:info).with(%r{Migrated #{success}/#{total} files})
+
+          perform
+        end
+      end
+
+      if failures > 0
+        it 'outputs upload failures' do
+          expect(Rails.logger).to receive(:warn).with(/Error .* I am a teapot/)
+
+          perform
+        end
+      end
+    end
+
+    it_behaves_like 'outputs correctly', success: 10
+
+    it 'migrates files' do
+      perform
+
+      aggregate_failures do
+        projects.each do |project|
+          expect(project.reload.avatar.upload.local?).to be_falsey
+        end
+      end
+    end
+
+    context 'migration is unsuccessful' do
+      before do
+        allow_any_instance_of(ObjectStorage::Concern).to receive(:migrate!).and_raise(CarrierWave::UploadError, "I am a teapot.")
+      end
+
+      it_behaves_like 'outputs correctly', failures: 10
+    end
+  end
+end