Documentation, start client work

.rvmrc

@@ -1 +1,2 @@
+export JRUBY_OPTS="--1.9"
 rvm use jruby@sidekiq --create

Changes.md

@@ -0,0 +1,4 @@
+HEAD
+-----------
+
+- Still working on an initial release!

Gemfile.lock

@@ -4,6 +4,8 @@ PATH
     sidekiq (0.1.0)
       celluloid
       connection_pool
+      multi_json
+      redis
 
 GEM
   remote: http://rubygems.org/
@@ -11,10 +13,13 @@ GEM
     celluloid (0.7.2)
     connection_pool (0.1.0)
     minitest (2.10.0)
+    multi_json (1.0.4)
     rake (0.9.2.2)
+    redis (2.2.2)
 
 PLATFORMS
   java
+  ruby
 
 DEPENDENCIES
   minitest

README.md

@@ -1,4 +1,106 @@
 Sidekiq
----------------
+==============
 
-It's an obscure project you've never heard of.
+Simple, efficient message processing for Ruby.
+
+Sidekiq aims to be a drop-in replacement for Resque.  It uses the exact same
+message format as Resque so it can slowly replace an existing Resque processing farm.
+You can have Sidekiq and Resque run side-by-side at the same time and
+use the Resque client to enqueue messages in Redis to be
+processed by Sidekiq.
+
+Sidekiq is different from Resque in how it processes messages: it
+processes many messages concurrently per process.  Resque only processes
+one message at a time per process so it is far less memory efficient.
+You'll find that you might need 50 200MB resque processes to peg your CPU
+whereas one 300MB Sidekiq process will peg the same CPU and perform the
+same amount of work.  Please see [my blog post on Resque's memory
+efficiency](http://blog.carbonfive.com/2011/09/16/improving-resques-memory-efficiency/)
+ and how I was able to shrink a Carbon Five client's processing farm
+from 9 machines to 1 machine.
+
+
+Requirements
+-----------------
+
+I test on Ruby 1.9.3 and JRuby 1.6.5 in 1.9 mode.  Other versions/VMs are
+untested.
+
+
+Installation
+-----------------
+
+   gem install sidekiq
+
+
+Usage
+-----------------
+
+See `sidekiq -h` for usage details.
+
+
+Client
+-----------------
+
+The Sidekiq client can be used to enqueue messages for processing:
+
+    Sidekiq::Client.push('some_queue', :class => SomeWorker, :args => ['bob', 2, foo: 'bar'])
+
+
+How it works
+-----------------
+
+Sidekiq assumes you are running a Rails 3 application with workers in app/workers.  Each message has a format like:
+
+    { class: 'SomeWorker', args: ['bob', 2, {foo: 'bar'}] }
+
+Sidekiq will instantiate a new instance of SomeWorker and call perform
+with args splatted:
+
+    class SomeWorker
+      def perform(name, count, options)
+      end
+    end
+
+This is the main API difference between Resque and Sidekiq: the perform
+method is an *instance* method, not a *class* method.
+
+
+Connections
+-----------------
+
+If your workers are connecting to mongo, memcached, redis, cassandra,
+etc you might want to set up a shared connection pool that all workers
+can use so you aren't opening a new connection for every message
+processed.  Sidekiq contains a connection pool API which you can use in your code to
+ensure safe, simple access to shared IO connections.  Please see the
+[connection\_pool gem](https://github.com/mperham/connection_pool) for more information.
+Your worker would do something like this:
+
+    class Worker
+      REDIS_POOL = ConnectionPool.new(:size => 10, :timeout => 3) { Redis.new }
+      def perform(args)
+        REDIS_POOL.with_connection do |redis|
+          redis.lsize(:foo)
+        end
+      end
+    end
+
+This ensures that if you have a concurrency setting of 50, you'll still only
+have a maximum of 10 connections open to Redis.
+
+
+Error Handling
+-----------------
+
+Sidekiq has built-in support for Airbrake.  If a worker raises an
+exception, Sidekiq will optionally send that error with the message
+context to Airbrake, log the error and then replace the worker with a
+fresh worker.  Just make sure you have Airbrake configured in your Rails
+app.
+
+
+Author
+-----------------
+
+Mike Perham, [@mperham](https://twitter.com/mperham), [http://mikeperham.com](http://mikeperham.com)

Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.warning = true
+  test.pattern = 'test/**/test_*.rb'
+end
+
+task :default => :test

TODO.md

@@ -0,0 +1,2 @@
+- monitoring hooks
+- resque-ui web ui?

lib/sidekiq/cli.rb

@@ -26,7 +26,7 @@ module Sidekiq
     private
 
     def enable_rails3
-      APP_PATH = File.expand_path('config/application.rb')
+      #APP_PATH = File.expand_path('config/application.rb')
       require File.expand_path('config/boot.rb')
     end
 
@@ -47,10 +47,12 @@ module Sidekiq
 
     def parse_options(argv=ARGV)
       @options = {
+        :daemon => false,
         :verbose => false,
         :queues => [],
         :worker_count => 25,
-        :server => 'localhost:6379'
+        :server => 'localhost:6379',
+        :pidfile => nil,
       }
 
       @parser = OptionParser.new do |o|
@@ -61,6 +63,10 @@ module Sidekiq
           end
         end
 
+        o.on "-d", "Daemonize" do |arg|
+          @options[:daemon] = arg
+        end
+
         o.on "--pidfile PATH", "Use PATH as a pidfile" do |arg|
           @options[:pidfile] = arg
         end
@@ -73,12 +79,12 @@ module Sidekiq
           @options[:server] = arg
         end
 
-        o.on '-c', '--count INT', "worker count to use" do |arg|
+        o.on '-c', '--concurrency INT', "Worker threads to use" do |arg|
           @options[:worker_count] = arg.to_i
         end
       end
 
-      @parser.banner = "sidekiq -q foo,1 -q bar,2 <options>"
+      @parser.banner = "sidekiq -q foo,1 -q bar,2 <more options>"
       @parser.on_tail "-h", "--help", "Show help" do
         log @parser
         exit 1

lib/sidekiq/client.rb

@@ -0,0 +1,41 @@
+require 'multi_json'
+
+module Sidekiq
+  class Client
+
+    def self.redis
+      @redis ||= Redis.new
+    end
+
+    def self.redis=(redis)
+      @redis = redis
+    end
+
+    # Example usage:
+    # Sidekiq::Client.push('my_queue', :class => MyWorker, :args => ['foo', 1, :bat => 'bar'])
+    def self.push(queue, item)
+      raise(ArgumentError, "Message must be a Hash of the form: { :class => SomeClass, :args => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item[:class] || !item[:args]
+
+      item[:class] = item[:class].to_s if !item[:class].is_a?(String)
+      redis.rpush("queue:#{queue}", MultiJson.encode(item))
+    end
+
+    # Please use .push if possible instead.
+    #
+    # Example usage:
+    #
+    #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+    #
+    # where MyWorker has defined:
+    #
+    #   def self.queue
+    #     'my_queue'
+    #   end
+    #
+    def self.enqueue(klass, *args)
+      queue = klass.instance_variable_get(:@queue) || (klass.respond_to?(:queue) && klass.queue) || raise(ArgumentError, "Cannot determine queue to use")
+      push(queue, { 'class' => klass.name, 'args' => args })
+    end
+  end
+end

lib/sidekiq/worker.rb

@@ -2,9 +2,20 @@ module Sidekiq
   class Worker
     include Celluloid
 
-    def process(json)
+    def process(hash)
-      klass = json['class'].constantize
+      begin
-      klass.new.perform(*json['args'])
+        klass = hash['class'].constantize
+        klass.new.perform(*hash['args'])
+      rescue => ex
+        airbrake(json, ex) if defined?(::Airbrake)
+        raise ex
+      end
+    end
+
+    def airbrake(json, ex)
+      ::Airbrake.notify(:error_class   => ex.class.name,
+                        :error_message => "#{ex.class.name}: #{e.message}",
+                        :parameters    => json)
     end
   end
 end

sidekiq.gemspec

@@ -13,8 +13,10 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
+  gem.add_dependency                  'redis'
   gem.add_dependency                  'connection_pool'
   gem.add_dependency                  'celluloid'
+  gem.add_dependency                  'multi_json'
   gem.add_development_dependency      'minitest'
   gem.add_development_dependency      'rake'
 end

test/helper.rb

@@ -0,0 +1,3 @@
+require 'minitest/unit'
+require 'minitest/pride'
+require 'minitest/autorun'

test/test_client.rb

@@ -0,0 +1,17 @@
+require 'helper'
+require 'sidekiq/client'
+
+class TestClient < MiniTest::Unit::TestCase
+  def test_argument_handling
+    assert_raises ArgumentError do
+      Sidekiq::Client.push('foo', 1)
+    end
+
+    assert_raises ArgumentError do
+      Sidekiq::Client.push('foo', :class => 'Foo', :noargs => [1, 2])
+    end
+
+    count = Sidekiq::Client.push('foo', :class => 'Foo', :args => [1, 2])
+    assert count > 0
+  end
+end