From 50c4c35bdea5b3c4fa47fc17b2d5aadd54793d73 Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Sat, 22 Mar 2014 00:18:33 -0400
Subject: [PATCH 1/5] Change some multis which just do reads to be pipelined
 for fasbetter perf.

---
 lib/sidekiq/api.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb
index 0557e6e1..3615d04d 100644
--- a/lib/sidekiq/api.rb
+++ b/lib/sidekiq/api.rb
@@ -498,7 +498,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         procs = conn.smembers('processes')
         procs.sort.each do |key|
-          valid, workers = conn.multi do
+          valid, workers = conn.pipelined do
             conn.exists(key)
             conn.hgetall("#{key}:workers")
           end
@@ -521,7 +521,7 @@ module Sidekiq
         procs = conn.smembers('processes')
         return 0 if procs.empty?
 
-        conn.multi do
+        conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'busy')
           end

From e01675980d8cb580e717b0318bb00c19a4f26ddd Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Sat, 22 Mar 2014 00:24:19 -0400
Subject: [PATCH 2/5] Rewrite ProcessSet#each to use pipelining to avoid lots
 of roundtrips in large Sidekiq installations.

---
 lib/sidekiq/api.rb | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb
index 3615d04d..925348df 100644
--- a/lib/sidekiq/api.rb
+++ b/lib/sidekiq/api.rb
@@ -447,13 +447,22 @@ module Sidekiq
       procs = Sidekiq.redis { |conn| conn.smembers('processes') }
 
       to_prune = []
+      sorted = procs.sort
       Sidekiq.redis do |conn|
-        procs.sort.each do |key|
-          info, busy, at_s = conn.hmget(key, 'info', 'busy', 'beat')
+        # We're making a tradeoff here between consuming more memory instead of
+        # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
+        # you'll be happier this way
+        result = conn.pipelined do
+          sorted.each do |key|
+            conn.hmget(key, 'info', 'busy', 'beat')
+          end
+        end
+
+        result.each_with_index do |(info, busy, at_s), i|
           # the hash named key has an expiry of 60 seconds.
           # if it's not found, that means the process has not reported
           # in to Redis and probably died.
-          (to_prune << key; next) if info.nil?
+          (to_prune << sorted[i]; next) if info.nil?
           hash = Sidekiq.load_json(info)
           yield hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f)
         end

From d0d94e876a9cc863fb1be310e2a9a2dbf383a3db Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Sat, 22 Mar 2014 00:28:27 -0400
Subject: [PATCH 3/5] Use mget in History instead of making many roundtrips.

---
 lib/sidekiq/api.rb | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb
index 925348df..139a2fac 100644
--- a/lib/sidekiq/api.rb
+++ b/lib/sidekiq/api.rb
@@ -68,15 +68,19 @@ module Sidekiq
       def date_stat_hash(stat)
         i = 0
         stat_hash = {}
+        keys = []
+        dates = []
+
+        while i < @days_previous
+          date = @start_date - i
+          keys << "stat:#{stat}:#{date}"
+          dates << date
+          i += 1
+        end
 
         Sidekiq.redis do |conn|
-          while i < @days_previous
-            date = @start_date - i
-            value = conn.get("stat:#{stat}:#{date}")
-
-            stat_hash[date.to_s] = value ? value.to_i : 0
-
-            i += 1
+          conn.mget(keys).each_with_index do |value, i|
+            stat_hash[dates[i].to_s] = value ? value.to_i : 0
           end
         end
 

From b47bbbc2a6a81d67af25cef36b83c75b4867eeb8 Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Sat, 22 Mar 2014 00:30:22 -0400
Subject: [PATCH 4/5] Pipeline queue lengths.

---
 lib/sidekiq/api.rb | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb
index 139a2fac..f5f3b53d 100644
--- a/lib/sidekiq/api.rb
+++ b/lib/sidekiq/api.rb
@@ -24,8 +24,16 @@ module Sidekiq
       Sidekiq.redis do |conn|
         queues = conn.smembers('queues')
 
+        lengths = conn.pipelined do
+          queues.each do |queue|
+            conn.llen("queue:#{queue}")
+          end
+        end
+
+        i = 0
         array_of_arrays = queues.inject({}) do |memo, queue|
-          memo[queue] = conn.llen("queue:#{queue}")
+          memo[queue] = lengths[i]
+          i += 1
           memo
         end.sort_by { |_, size| size }
 

From ff9475096c280d690b703cbef8625003ce9420ba Mon Sep 17 00:00:00 2001
From: Jonathan Hyman <jonathan.hyman@post.harvard.edu>
Date: Sat, 22 Mar 2014 00:31:50 -0400
Subject: [PATCH 5/5] Use mset instead of a loop over set for Stats#reset.

---
 lib/sidekiq/api.rb | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb
index f5f3b53d..f0b741c4 100644
--- a/lib/sidekiq/api.rb
+++ b/lib/sidekiq/api.rb
@@ -15,8 +15,13 @@ module Sidekiq
       all   = %w(failed processed)
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
 
+      mset_args = []
+      stats.each do |stat|
+        mset_args << "stat:#{stat}"
+        mset_args << 0
+      end
       Sidekiq.redis do |conn|
-        stats.each { |stat| conn.set("stat:#{stat}", 0) }
+        conn.mset(*mset_args)
       end
     end