Disconnect and retry if READONLY error detected, fixes #2550

lib/sidekiq.rb

@@ -76,9 +76,19 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" unless block
-    redis_pool.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
   def self.redis_pool

test/test_sidekiq.rb

@@ -84,4 +84,24 @@ class TestSidekiq < Sidekiq::Test
       assert_includes output, "ERROR"
     end
   end
+
+  describe 'redis connection' do
+    it 'does not continually retry' do
+      assert_raises Redis::CommandError do
+        Sidekiq.redis do |c|
+          raise Redis::CommandError, "READONLY You can't write against a read only slave."
+        end
+      end
+    end
+
+    it 'reconnects if connection is flagged as readonly' do
+      counts = []
+      Sidekiq.redis do |c|
+        counts << c.info['total_connections_received'].to_i
+        raise Redis::CommandError, "READONLY You can't write against a read only slave." if counts.size == 1
+      end
+      assert_equal 2, counts.size
+      assert_equal counts[0] + 1, counts[1]
+    end
+  end
 end