Compress jobs backtraces

Changes.md

@@ -5,6 +5,7 @@
 HEAD
 ---------
 
+- Compress jobs backtraces before pushing into Redis [#4272]
 - Support display of ActiveJob 6.0 payloads in the Web UI [#4263]
 - Add `SortedSet#scan` for pattern based scanning. For large sets this API will be **MUCH** faster
   than standard iteration using each.

lib/sidekiq/api.rb

@@ -2,6 +2,9 @@
 
 require "sidekiq"
 
+require "zlib"
+require "base64"
+
 module Sidekiq
   class Stats
     def initialize
@@ -385,6 +388,16 @@ module Sidekiq
       Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
     end
 
+    def error_backtrace
+      # Cache nil values
+      if defined?(@error_backtrace)
+        @error_backtrace
+      else
+        value = self["error_backtrace"]
+        @error_backtrace = value && uncompress_backtrace(value)
+      end
+    end
+
     attr_reader :queue
 
     def latency
@@ -418,6 +431,17 @@ module Sidekiq
       Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
+
+    def uncompress_backtrace(backtrace)
+      if backtrace.is_a?(Array)
+        # Handle old jobs with previous backtrace format
+        backtrace
+      else
+        decoded = Base64.decode64(backtrace)
+        uncompressed = Zlib::Inflate.inflate(decoded)
+        Marshal.load(uncompressed)
+      end
+    end
   end
 
   class SortedEntry < Job

lib/sidekiq/job_retry.rb

@@ -3,6 +3,9 @@
 require "sidekiq/scheduled"
 require "sidekiq/api"
 
+require "zlib"
+require "base64"
+
 module Sidekiq
   ##
   # Automatically retry jobs that fail in Sidekiq.
@@ -151,12 +154,14 @@ module Sidekiq
         msg["retry_count"] = 0
       end
 
-      if msg["backtrace"] == true
-        msg["error_backtrace"] = exception.backtrace
-      elsif !msg["backtrace"]
-        # do nothing
-      elsif msg["backtrace"].to_i != 0
-        msg["error_backtrace"] = exception.backtrace[0...msg["backtrace"].to_i]
+      if msg["backtrace"]
+        lines = if msg["backtrace"] == true
+          exception.backtrace
+        else
+          exception.backtrace[0...msg["backtrace"].to_i]
+        end
+
+        msg["error_backtrace"] = compress_backtrace(lines)
       end
 
       if count < max_retry_attempts
@@ -245,5 +250,11 @@ module Sidekiq
     rescue
       +"!!! ERROR MESSAGE THREW AN ERROR !!!"
     end
+
+    def compress_backtrace(backtrace)
+      serialized = Marshal.dump(backtrace)
+      compressed = Zlib::Deflate.deflate(serialized)
+      Base64.encode64(compressed)
+    end
   end
 end

test/test_api.rb

@@ -254,6 +254,12 @@ describe 'API' do
       assert_equal [1,2,3], x.display_args
     end
 
+    it 'handles old jobs error_backtrace format' do
+      add_retry
+      job = Sidekiq::RetrySet.new.first
+      assert_equal ['line1', 'line2'], job.error_backtrace
+    end
+
     describe "Rails unwrapping" do
       SERIALIZED_JOBS = {
         "5.x" => [
@@ -587,7 +593,7 @@ describe 'API' do
     end
 
     def add_retry(jid = 'bob', at = Time.now.to_f)
-      payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
+      payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f, 'error_backtrace' => ['line1', 'line2'])
       Sidekiq.redis do |conn|
         conn.zadd('retry', at.to_s, payload)
       end

test/test_retry.rb

@@ -3,6 +3,7 @@
 require_relative 'helper'
 require 'sidekiq/scheduled'
 require 'sidekiq/job_retry'
+require 'sidekiq/api'
 
 describe Sidekiq::JobRetry do
   describe 'middleware' do
@@ -116,8 +117,10 @@ describe Sidekiq::JobRetry do
           c = caller(0); raise "kerblammo!"
         end
       end
-      assert job["error_backtrace"]
-      assert_equal c[0], job["error_backtrace"][0]
+
+      job = Sidekiq::RetrySet.new.first
+      assert job.error_backtrace
+      assert_equal c[0], job.error_backtrace[0]
     end
 
     it 'saves partial backtraces' do
@@ -127,8 +130,10 @@ describe Sidekiq::JobRetry do
           c = caller(0)[0...3]; raise "kerblammo!"
         end
       end
-      assert job["error_backtrace"]
-      assert_equal c, job["error_backtrace"]
+
+      job = Sidekiq::RetrySet.new.first
+      assert job.error_backtrace
+      assert_equal c, job.error_backtrace
       assert_equal 3, c.size
     end
 

web/views/dead.erb

@@ -14,11 +14,11 @@
         <th><%= t('ErrorMessage') %></th>
         <td><%= h(@dead['error_message']) %></td>
       </tr>
-      <% if !@dead['error_backtrace'].nil? %>
+      <% if @dead.error_backtrace %>
         <tr>
           <th><%= t('ErrorBacktrace') %></th>
           <td>
-            <code><%= @dead['error_backtrace'].join("<br/>") %></code>
+            <code><%= @dead.error_backtrace.join("<br/>") %></code>
           </td>
         </tr>
       <% end %>

web/views/retry.erb

@@ -14,11 +14,11 @@
         <th><%= t('ErrorMessage') %></th>
         <td><%= h(@retry['error_message']) %></td>
       </tr>
-      <% if !@retry['error_backtrace'].nil? %>
+      <% if @retry.error_backtrace %>
         <tr>
           <th><%= t('ErrorBacktrace') %></th>
           <td>
-            <code><%= @retry['error_backtrace'].join("<br/>") %></code>
+            <code><%= @retry.error_backtrace.join("<br/>") %></code>
           </td>
         </tr>
       <% end %>