Merge pull request #780 from EasyGive/master

Hook to process events after a job fails all retries

lib/sidekiq/middleware/server/retry_jobs.rb

@@ -14,7 +14,9 @@ module Sidekiq
       # 3. after a few days, a developer deploys a fix.  the message is
       #    reprocessed successfully.
       # 4. if 3 never happens, sidekiq will eventually give up and throw the
-      #    message away.
+      #    message away. If the worker defines a method called 'retries_exhausted',
+      #    this will be called before throwing the message away. If the
+      #    'retries_exhausted' method throws an exception, it's dropped and logged.
       #
       # A message looks like:
       #
@@ -81,11 +83,20 @@ module Sidekiq
             end
           else
             # Goodbye dear message, you (re)tried your best I'm sure.
-            logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+            retries_exhausted(msg)
           end
+
           raise e
         end
 
+        def retries_exhausted(msg)
+          logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+          worker.retries_exhausted(*msg['args']) if worker.respond_to?(:retries_exhausted)
+
+        rescue Exception => e
+          logger.debug { "Failure during `retries_exhausted` hook: #{e} - #{msg}" }
+        end
+
         def retry_attempts_from(msg_retry, default)
           if msg_retry.is_a?(Fixnum)
             msg_retry

test/test_retry.rb

@@ -167,6 +167,34 @@ class TestRetry < MiniTest::Unit::TestCase
       # MiniTest can't assert that a method call did NOT happen!?
       assert_raises(MockExpectationError) { @redis.verify }
     end
+
+    describe "retry exhaustion" do
+      let(:worker){ MiniTest::Mock.new }
+      let(:handler){ Sidekiq::Middleware::Server::RetryJobs.new }
+      let(:msg){ {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>Time.now.utc, "retry"=>3, "retry_count"=>3} }
+
+      it 'calls worker retries_exhausted after too many retries' do
+        worker.expect(:retries_exhausted, true, [1,2,3]) 
+        task_misbehaving_worker
+      end
+
+      it 'handles and logs retries_exhausted failures gracefully (drops them)' do
+        def worker.retries_exhausted(*args)
+          raise 'bam!'
+        end
+
+        e = task_misbehaving_worker
+        assert_equal e.message, "kerblammo!"
+      end
+
+      def task_misbehaving_worker
+        assert_raises RuntimeError do
+          handler.call('', msg, 'default') do
+            raise 'kerblammo!'
+          end
+        end
+      end
+    end
   end
 
   describe 'poller' do