Catch exceptions when stealing background migrations

lib/gitlab/background_migration.rb

@@ -19,7 +19,12 @@ module Gitlab
           next unless job.queue == self.queue
           next unless migration_class == steal_class
 
+          begin
             perform(migration_class, migration_args) if job.delete
+          rescue => e
+            Logger.new($stdout).warn(e.message)
+            next
+          end
         end
       end
     end

spec/lib/gitlab/background_migration_spec.rb

@@ -10,18 +10,17 @@ describe Gitlab::BackgroundMigration do
 
   describe '.steal' do
     context 'when there are enqueued jobs present' do
-      let(:job) { double(:job, args: ['Foo', [10, 20]]) }
-      let(:queue) { [job] }
+      let(:queue) do
+        [double(args: ['Foo', [10, 20]], queue: described_class.queue)]
+      end
 
       before do
         allow(Sidekiq::Queue).to receive(:new)
           .with(described_class.queue)
           .and_return(queue)
-
-        allow(job).to receive(:queue)
-          .and_return(described_class.queue)
       end
 
+      context 'when queue contains unprocessed jobs' do
         it 'steals jobs from a queue' do
           expect(queue[0]).to receive(:delete).and_return(true)
 
@@ -47,6 +46,32 @@ describe Gitlab::BackgroundMigration do
         end
       end
 
+      context 'when one of the jobs raises an error' do
+        let(:migration) { spy(:migration) }
+
+        let(:queue) do
+          [double(args: ['Foo', [10, 20]], queue: described_class.queue),
+           double(args: ['Foo', [20, 30]], queue: described_class.queue)]
+        end
+
+        before do
+          stub_const("#{described_class}::Foo", migration)
+
+          allow(migration).to receive(:perform).with(10, 20)
+            .and_raise(StandardError, 'Migration error')
+
+          allow(queue[0]).to receive(:delete).and_return(true)
+          allow(queue[1]).to receive(:delete).and_return(true)
+        end
+
+        it 'recovers from an exceptions and continues' do
+          expect(migration).to receive(:perform).twice
+          expect { described_class.steal('Foo') }
+            .to output(/Migration error/).to_stdout
+        end
+      end
+    end
+
     context 'when there are scheduled jobs present', :sidekiq, :redis do
       it 'steals all jobs from the scheduled sets' do
         Sidekiq::Testing.disable! do