Add dead job queue, fixes #1471

Changes.md

@@ -1,6 +1,9 @@
 3.0.0
 -----------
 
+- Dead Job Queue - jobs which run out of retries are now moved to a dead
+  job queue.  These jobs must be retried manually or they will expire
+  after 6 months or 10,000 jobs.
 - **Remove official support for Ruby 1.9**  Things still might work but
   I no longer actively test on it.
 - Remove built-in support for Redis-to-Go.

lib/sidekiq/api.rb

@@ -44,6 +44,10 @@ module Sidekiq
       Sidekiq.redis {|c| c.zcard('retry') }
     end
 
+    def dead_size
+      Sidekiq.redis {|c| c.zcard('dead') }
+    end
+
     class History
       def initialize(days_previous, start_date = nil)
         @days_previous = days_previous
@@ -392,6 +396,18 @@ module Sidekiq
     end
   end
 
+  class DeadSet < SortedSet
+    def initialize
+      super 'dead'
+    end
+
+    def retry_all
+      while size > 0
+        each(&:retry)
+      end
+    end
+  end
+
 
   ##
   # Programmatic access to the current active worker set.

lib/sidekiq/middleware/server/retry_jobs.rb

@@ -108,10 +108,23 @@ module Sidekiq
 
         private
 
+        DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
+        MAX_JOBS = 10_000
+
         def retries_exhausted(worker, msg)
           logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
           if worker.sidekiq_retries_exhausted_block?
             worker.sidekiq_retries_exhausted_block.call(msg)
+          else
+            Sidekiq.logger.info { "Adding a dead #{msg['class']} job" }
+            payload = Sidekiq.dump_json(msg)
+            Sidekiq.redis do |conn|
+              conn.multi do
+                conn.zadd('dead', Time.now.to_f, payload)
+                conn.zremrangebyscore('dead', '-inf', (Time.now.to_i - DEAD_JOB_TIMEOUT).to_f)
+                conn.zremrangebyrank('dead', 0, -MAX_JOBS)
+              end
+            end
           end
         rescue Exception => e
           handle_exception(e, { :context => "Error calling retries_exhausted" })

lib/sidekiq/web.rb

@@ -24,6 +24,7 @@ module Sidekiq
       "Queues"    => 'queues',
       "Retries"   => 'retries',
       "Scheduled" => 'scheduled',
+      "Dead"      => 'morgue',
     }
 
     class << self
@@ -70,6 +71,59 @@ module Sidekiq
       redirect_with_query("#{root_path}queues/#{params[:name]}")
     end
 
+    get '/morgue' do
+      @count = (params[:count] || 25).to_i
+      (@current_page, @total_size, @dead) = page("dead", params[:page], @count)
+      @dead = @dead.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      erb :morgue
+    end
+
+    get "/morgue/:key" do
+      halt 404 unless params['key']
+      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
+      redirect "#{root_path}morgue" if @dead.nil?
+      erb :dead
+    end
+
+    post '/morgue' do
+      halt 404 unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+        next unless job
+        if params['retry']
+          job.retry
+        elsif params['delete']
+          job.delete
+        end
+      end
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    post "/morgue/all/delete" do
+      Sidekiq::DeadSet.new.clear
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/all/retry" do
+      Sidekiq::DeadSet.new.retry_all
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/:key" do
+      halt 404 unless params['key']
+      job = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
+      if job
+        if params['retry']
+          job.retry
+        elsif params['delete']
+          job.delete
+        end
+      end
+      redirect_with_query("#{root_path}morgue")
+    end
+
+
     get '/retries' do
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @retries) = page("retry", params[:page], @count)

test/test_web.rb

@@ -392,6 +392,19 @@ class TestWeb < Sidekiq::Test
       end
     end
 
+    describe 'dead jobs' do
+      it 'shows empty index' do
+        get 'morgue'
+        assert_equal 200, last_response.status
+      end
+      it 'shows index with jobs' do
+        (_, score) = add_dead
+        get 'morgue'
+        assert_equal 200, last_response.status
+        assert_match /#{score}/, last_response.body
+      end
+    end
+
     def add_scheduled
       score = Time.now.to_f
       msg = { 'class' => 'HardWorker',
@@ -419,6 +432,22 @@ class TestWeb < Sidekiq::Test
       [msg, score]
     end
 
+    def add_dead
+      msg = { 'class' => 'HardWorker',
+              'args' => ['bob', 1, Time.now.to_f],
+              'queue' => 'default',
+              'error_message' => 'Some fake message',
+              'error_class' => 'RuntimeError',
+              'retry_count' => 0,
+              'failed_at' => Time.now.utc,
+              'jid' => SecureRandom.hex(12) }
+      score = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.zadd('dead', score, Sidekiq.dump_json(msg))
+      end
+      [msg, score]
+    end
+
     def add_xss_retry
       msg = { 'class' => 'FailWorker',
               'args' => ['<a>hello</a>'],

web/locales/en.yml

@@ -61,3 +61,6 @@ en: # <---- change this to your locale code
   ThreeMonths: 3 months
   SixMonths: 6 months
   Failures: Failures
+  DeadJobs: Dead Jobs
+  NoDeadJobsFound: No dead jobs were found
+  Dead: Dead

web/views/_job_info.erb

@@ -72,5 +72,11 @@
         <td><%= relative_time(job.at) %></td>
       </tr>
     <% end %>
+    <% if type == :dead %>
+      <tr>
+        <th><%= t('LastRetry') %></th>
+        <td><%= relative_time(job.at) %></td>
+      </tr>
+    <% end %>
   </tbody>
 </table>

web/views/_summary.erb

@@ -1,34 +1,40 @@
 <ul class="list-unstyled summary row">
-  <li class="processed col-sm-2">
+  <li class="processed col-sm-1">
     <span class="count"><%= number_with_delimiter(stats.processed) %></span>
     <span class="desc"><%= t('Processed') %></span>
   </li>
-  <li class="failed col-sm-2">
+  <li class="failed col-sm-1">
     <span class="count"><%= number_with_delimiter(stats.failed) %></span>
     <span class="desc"><%= t('Failed') %></span>
   </li>
-  <li class="busy col-sm-2">
+  <li class="busy col-sm-1">
     <a href="<%= root_path %>workers">
       <span class="count"><%= number_with_delimiter(workers_size) %></span>
       <span class="desc"><%= t('Busy') %></span>
     </a>
   </li>
-  <li class="enqueued col-sm-2">
+  <li class="enqueued col-sm-1">
     <a href="<%= root_path %>queues">
       <span class="count"><%= number_with_delimiter(stats.enqueued) %></span>
       <span class="desc"><%= t('Enqueued') %></span>
     </a>
   </li>
-  <li class="retries col-sm-2">
+  <li class="retries col-sm-1">
     <a href="<%= root_path %>retries">
       <span class="count"><%= number_with_delimiter(stats.retry_size) %></span>
       <span class="desc"><%= t('Retries') %></span>
     </a>
   </li>
-  <li class="scheduled col-sm-2">
+  <li class="scheduled col-sm-1">
     <a href="<%= root_path %>scheduled">
       <span class="count"><%= number_with_delimiter(stats.scheduled_size) %></span>
       <span class="desc"><%= t('Scheduled') %></span>
     </a>
   </li>
+  <li class="dead col-sm-1">
+    <a href="<%= root_path %>morgue">
+      <span class="count"><%= number_with_delimiter(stats.dead_size) %></span>
+      <span class="desc"><%= t('Dead') %></span>
+    </a>
+  </li>
 </ul>

web/views/dead.erb

@@ -0,0 +1,30 @@
+<%= erb :_job_info, :locals => {:job => @dead, :type => :dead} %>
+
+<h3><%= t('Error') %></h3>
+<table class="error table table-bordered table-striped">
+  <tbody>
+    <tr>
+      <th><%= t('ErrorClass') %></th>
+      <td>
+        <code><%= @dead['error_class'] %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('ErrorMessage') %></th>
+      <td><%= h(@dead['error_message']) %></td>
+    </tr>
+    <% if !@dead['error_backtrace'].nil? %>
+      <tr>
+        <th><%= t('ErrorBacktrace') %></th>
+        <td>
+          <code><%= @dead['error_backtrace'].join("<br/>") %></code>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<form class="form-horizontal" action="<%= root_path %>morgue/<%= job_params(@dead, @dead.score) %>" method="post">
+  <a class="btn btn-default" href="<%= root_path %>morgue"><%= t('GoBack') %></a>
+  <input class="btn btn-primary" type="submit" name="retry" value="<%= t('RetryNow') %>" />
+  <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" />
+</form>

web/views/morgue.erb

@@ -0,0 +1,66 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3><%= t('DeadJobs') %></h3>
+  </div>
+  <% if @dead.size > 0 && @total_size > @count %>
+    <div class="col-sm-4">
+      <%= erb :_paging, :locals => { :url => "#{root_path}morgue" } %>
+    </div>
+  <% end %>
+  <%= filtering('dead') %>
+</header>
+
+<% if @dead.size > 0 %>
+  <form action="<%= root_path %>morgue" method="post">
+    <table class="table table-striped table-bordered table-white">
+      <thead>
+        <tr>
+          <th width="20px" class="table-checkbox">
+            <label>
+              <input type="checkbox" class="check_all" />
+            </label>
+          </th>
+          <th width="25%"><%= t('LastRetry') %></th>
+          <th><%= t('Queue') %></th>
+          <th><%= t('Worker') %></th>
+          <th><%= t('Arguments') %></th>
+          <th><%= t('Error') %></th>
+        </tr>
+      </thead>
+      <% @dead.each do |entry| %>
+        <tr>
+          <td class="table-checkbox">
+            <label>
+              <input type='checkbox' name='key[]' value='<%= job_params(entry.item, entry.score) %>' />
+            </label>
+          </td>
+          <td>
+            <a href="<%= root_path %>morgue/<%= job_params(entry.item, entry.score) %>"><%= relative_time(entry.at) %></a>
+          </td>
+          <td>
+            <a href="<%= root_path %>queues/<%= entry.queue %>"><%= entry.queue %></a>
+          </td>
+          <td><%= entry.klass %></td>
+          <td>
+            <div class="args"><%= display_args(entry.args) %></div>
+          </td>
+          <td>
+            <div><%= h truncate("#{entry['error_class']}: #{entry['error_message']}", 200) %></div>
+          </td>
+        </tr>
+      <% end %>
+    </table>
+    <input class="btn btn-primary btn-xs pull-left" type="submit" name="retry" value="<%= t('RetryNow') %>" />
+    <input class="btn btn-danger btn-xs pull-left" type="submit" name="delete" value="<%= t('Delete') %>" />
+  </form>
+
+  <form action="<%= root_path %>morgue/all/delete" method="post">
+    <input class="btn btn-danger btn-xs pull-right" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+  <form action="<%= root_path %>morgue/all/retry" method="post">
+    <input class="btn btn-danger btn-xs pull-right" type="submit" name="retry" value="<%= t('RetryAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+
+<% else %>
+  <div class="alert alert-success"><%= t('NoDeadJobsFound') %></div>
+<% end %>