Add global failure handlers, fixes #3721

Changes.md

@@ -5,7 +5,10 @@
 5.1.0
 -----------
 
-- Enable ActiveRecord query cache within jobs by default [#3718, sobrinho]
+- **NEW** Global failure handlers - called when your job exhausts all
+  retries and dies.  Now you can take action when a job fails permanently. [#3721]
+- **NEW** Enable ActiveRecord query cache within jobs by default [#3718, sobrinho]
+  This will prevent duplicate SELECTS; cache is cleared upon any UPDATE/INSERT/DELETE.
   See the issue for how to bypass the cache or disable it completely.
 - Exceptions during the :startup event will now kill the process [#3717]
 - Make `Sidekiq::Client.via` reentrant [#3715]

lib/sidekiq.rb

@@ -25,6 +25,7 @@ module Sidekiq
     poll_interval_average: nil,
     average_scheduled_poll_interval: 15,
     error_handlers: [],
+    failure_handlers: [],
     lifecycle_events: {
       startup: [],
       quiet: [],
@@ -156,16 +157,22 @@ module Sidekiq
     defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
   end
 
+  def self.default_retries_exhausted=(prok)
+    logger.info { "default_retries_exhausted is deprecated, please use `config.failure_handlers << -> {|job, ex| }`" }
+    failure_handlers << prok
+  end
+
+  ##
+  # A failure handler is called when all retries for a job have been exhausted and
+  # the job must be killed or dropped.  It's the notification to your application
+  # that this job will not finish without manual intervention.
+  #
   # Sidekiq.configure_server do |config|
-  #   config.default_retries_exhausted = -> (job, ex) do
+  #   config.failure_handlers << -> (job, ex) do
   #   end
   # end
-  def self.default_retries_exhausted=(prok)
-    @default_retries_exhausted = prok
-  end
-  @default_retries_exhausted = ->(job, ex) { }
-  def self.default_retries_exhausted
-    @default_retries_exhausted
+  def self.failure_handlers
+    options[:failure_handlers]
   end
 
   def self.load_json(string)

lib/sidekiq/job_retry.rb

@@ -172,10 +172,18 @@ module Sidekiq
     def retries_exhausted(worker, msg, exception)
       logger.debug { "Retries exhausted for job" }
       begin
-        block = worker && worker.sidekiq_retries_exhausted_block || Sidekiq.default_retries_exhausted
+        block = worker && worker.sidekiq_retries_exhausted_block
         block.call(msg, exception) if block
       rescue => e
-        handle_exception(e, { context: "Error calling retries_exhausted for #{msg['class']}", job: msg })
+        handle_exception(e, { context: "Error calling retries_exhausted", job: msg })
+      end
+
+      Sidekiq.failure_handlers.each do |handler|
+        begin
+          handler.call(msg, exception)
+        rescue => e
+          handle_exception(e, { context: "Error calling retries_exhausted", job: msg })
+        end
       end
 
       send_to_morgue(msg) unless msg['dead'] == false

test/test_retry_exhausted.rb

@@ -148,5 +148,33 @@ class TestRetryExhausted < Sidekiq::Test
         Sidekiq.default_retries_exhausted = nil
       end
     end
+
+    it 'allows global failure handlers' do
+      begin
+        class Foobar
+          include Sidekiq::Worker
+        end
+
+        exhausted_job = nil
+        exhausted_exception = nil
+        Sidekiq.failure_handlers.clear
+        Sidekiq.failure_handlers << proc do |job, ex|
+          exhausted_job = job
+          exhausted_exception = ex
+        end
+        f = Foobar.new
+        raised_error = assert_raises RuntimeError do
+          handler.local(f, job('retry_count' => 0, 'retry' => 1), 'default') do
+            raise 'kerblammo!'
+          end
+        end
+        raised_error = raised_error.cause
+
+        assert exhausted_job
+        assert_equal raised_error, exhausted_exception
+      ensure
+        Sidekiq.failure_handlers.clear
+      end
+    end
   end
 end