re-introduce weighted queues

lib/sidekiq/cli.rb

@@ -99,6 +99,7 @@ module Sidekiq
 
     def validate!
       options[:queues] << 'default' if options[:queues].empty?
+      options[:queues].shuffle!
 
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
@@ -115,8 +116,9 @@ module Sidekiq
       opts = {}
 
       @parser = OptionParser.new do |o|
-        o.on "-q", "--queue QUEUE", "Queue to process" do |arg|
+         o.on "-q", "--queue QUEUE,WEIGHT", "Queue to process, with optional weight" do |arg|
-          parse_queues(opts, arg)
+          q, weight = arg.split(",")
+          parse_queues(opts, q, weight)
         end
 
         o.on "-v", "--verbose", "Print more verbose output" do
@@ -175,14 +177,15 @@ module Sidekiq
       if cli[:config_file] && File.exist?(cli[:config_file])
         opts = YAML.load_file cli[:config_file]
         queues = opts.delete(:queues) || []
-        queues.each { |name, _| parse_queues(opts, name) }
+        queues.each { |name, weight| parse_queues(opts, name, weight) }
       end
       opts
     end
 
-    def parse_queues(opts, q)
+    def parse_queues(opts, q, weight)
+      (weight || 1).to_i.times do
        (opts[:queues] ||= []) << q
       end
-
+    end
   end
 end

lib/sidekiq/fetch.rb

@@ -10,14 +10,13 @@ module Sidekiq
     include Celluloid
     include Sidekiq::Util
 
+    # Timeout for Redis#blpop.
     TIMEOUT = 1
 
     def initialize(mgr, queues)
-      @cmd = queues.map { |q| "queue:#{q}" }
       @mgr = mgr
-
+      @queues = queues
-      # One second timeout for blpop.
+      @num_queues = queues.uniq.size
-      @cmd << TIMEOUT
     end
 
     # Fetching is straightforward: the Manager makes a fetch
@@ -32,7 +31,7 @@ module Sidekiq
       watchdog('Fetcher#fetch died') do
         queue = nil
         msg = nil
-        Sidekiq.redis { |conn| (queue, msg) = conn.blpop(*@cmd) }
+        Sidekiq.redis { |conn| queue, msg = conn.blpop(*queues_cmd) }
 
         if msg
           @mgr.assign!(msg, queue.gsub(/.*queue:/, ''))
@@ -42,5 +41,18 @@ module Sidekiq
       end
     end
 
+    private
+
+    # Creating the Redis#blpop command takes into account any
+    # configured queue weights. By default Redis#blpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#blpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      queues = @queues.sample(@num_queues)
+      cmd = queues.map { |q| "queue:#{q}" }
+      cmd << TIMEOUT
+      cmd
+    end
   end
 end

test/config.yml

@@ -5,5 +5,5 @@
 :pidfile:      /tmp/sidekiq-config-test.pid
 :concurrency:  50
 :queues:
-  - often
+  - [often, 2]
-  - seldom
+  - [seldom, 1]

test/test_cli.rb

@@ -45,9 +45,9 @@ class TestCli < MiniTest::Unit::TestCase
       assert_equal 30, Sidekiq.options[:timeout]
     end
 
-    it 'handles multiple queues' do
+    it 'handles multiple queues with weights' do
-      @cli.parse(['sidekiq', '-q', 'foo', '-q', 'bar', '-r', './test/fake_env.rb'])
+      @cli.parse(['sidekiq', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal %w(foo bar), Sidekiq.options[:queues]
+      assert_equal %w(bar foo foo foo), Sidekiq.options[:queues].sort
     end
 
     it 'sets verbose' do
@@ -110,7 +110,8 @@ class TestCli < MiniTest::Unit::TestCase
       end
 
       it 'sets queues' do
-        assert_equal %w(often seldom), Sidekiq.options[:queues]
+        assert_equal 2, Sidekiq.options[:queues].count { |q| q == 'often' }
+        assert_equal 1, Sidekiq.options[:queues].count { |q| q == 'seldom' }
       end
     end
 
@@ -132,8 +133,8 @@ class TestCli < MiniTest::Unit::TestCase
                     '-c', '100',
                     '-r', @tmp_lib_path,
                     '-P', @tmp_path,
-                    '-q', 'often',
+                    '-q', 'often,7',
-                    '-q', 'seldom'])
+                    '-q', 'seldom,3'])
       end
 
       after do
@@ -158,7 +159,8 @@ class TestCli < MiniTest::Unit::TestCase
       end
 
       it 'sets queues' do
-        assert_equal %w(often seldom), Sidekiq.options[:queues]
+        assert_equal 7, Sidekiq.options[:queues].count { |q| q == 'often' }
+        assert_equal 3, Sidekiq.options[:queues].count { |q| q == 'seldom' }
       end
     end
   end