From 79288de6058adc5f2aa6154953f60a67e188121f Mon Sep 17 00:00:00 2001
From: Mike Perham <mperham@gmail.com>
Date: Fri, 11 Sep 2015 10:55:45 -0700
Subject: [PATCH 1/2] Disconnect and retry if READONLY error detected, fixes
 #2550

---
 lib/sidekiq.rb       | 16 +++++++++++++---
 test/test_sidekiq.rb | 20 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/lib/sidekiq.rb b/lib/sidekiq.rb
index 1f1af22e..f3fbca6b 100644
--- a/lib/sidekiq.rb
+++ b/lib/sidekiq.rb
@@ -76,9 +76,19 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" unless block
-    redis_pool.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
   def self.redis_pool
diff --git a/test/test_sidekiq.rb b/test/test_sidekiq.rb
index ce167fd3..2b80f3a5 100644
--- a/test/test_sidekiq.rb
+++ b/test/test_sidekiq.rb
@@ -84,4 +84,24 @@ class TestSidekiq < Sidekiq::Test
       assert_includes output, "ERROR"
     end
   end
+
+  describe 'redis connection' do
+    it 'does not continually retry' do
+      assert_raises Redis::CommandError do
+        Sidekiq.redis do |c|
+          raise Redis::CommandError, "READONLY You can't write against a read only slave."
+        end
+      end
+    end
+
+    it 'reconnects if connection is flagged as readonly' do
+      counts = []
+      Sidekiq.redis do |c|
+        counts << c.info['total_connections_received'].to_i
+        raise Redis::CommandError, "READONLY You can't write against a read only slave." if counts.size == 1
+      end
+      assert_equal 2, counts.size
+      assert_equal counts[0] + 1, counts[1]
+    end
+  end
 end

From aff8531b02ca7b4003f1983fb6eda6912f60ecfd Mon Sep 17 00:00:00 2001
From: Mike Perham <mperham@gmail.com>
Date: Fri, 11 Sep 2015 11:03:01 -0700
Subject: [PATCH 2/2] changes

---
 Changes.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Changes.md b/Changes.md
index b97100e6..50521b90 100644
--- a/Changes.md
+++ b/Changes.md
@@ -1,7 +1,8 @@
-Next
+HEAD
 -----------
 
-- Add middleware stack to testing harness; see [wiki documentation](https://github.com/mperham/sidekiq/wiki/Testing#testing-server-middleware) [#2534, ryansch]
+- Disconnect and retry Redis operations if we see a READONLY error [#2550]
+- Add server middleware testing harness; see [wiki](https://github.com/mperham/sidekiq/wiki/Testing#testing-server-middleware) [#2534, ryansch]
 
 3.5.0
 -----------