diff --git a/.rvmrc b/.rvmrc index 8cb04084..138f3793 100644 --- a/.rvmrc +++ b/.rvmrc @@ -1 +1,2 @@ +export JRUBY_OPTS="--1.9" rvm use jruby@sidekiq --create diff --git a/Changes.md b/Changes.md index e69de29b..0c96d381 100644 --- a/Changes.md +++ b/Changes.md @@ -0,0 +1,4 @@ +HEAD +----------- + +- Still working on an initial release! diff --git a/Gemfile.lock b/Gemfile.lock index 677df4fc..eb9cc269 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,6 +4,8 @@ PATH sidekiq (0.1.0) celluloid connection_pool + multi_json + redis GEM remote: http://rubygems.org/ @@ -11,10 +13,13 @@ GEM celluloid (0.7.2) connection_pool (0.1.0) minitest (2.10.0) + multi_json (1.0.4) rake (0.9.2.2) + redis (2.2.2) PLATFORMS java + ruby DEPENDENCIES minitest diff --git a/README.md b/README.md index a3c45431..8fa74e24 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,106 @@ Sidekiq ---------------- +============== -It's an obscure project you've never heard of. +Simple, efficient message processing for Ruby. + +Sidekiq aims to be a drop-in replacement for Resque. It uses the exact same +message format as Resque so it can slowly replace an existing Resque processing farm. +You can have Sidekiq and Resque run side-by-side at the same time and +use the Resque client to enqueue messages in Redis to be +processed by Sidekiq. + +Sidekiq is different from Resque in how it processes messages: it +processes many messages concurrently per process. Resque only processes +one message at a time per process so it is far less memory efficient. +You'll find that you might need 50 200MB resque processes to peg your CPU +whereas one 300MB Sidekiq process will peg the same CPU and perform the +same amount of work. Please see [my blog post on Resque's memory +efficiency](http://blog.carbonfive.com/2011/09/16/improving-resques-memory-efficiency/) + and how I was able to shrink a Carbon Five client's processing farm +from 9 machines to 1 machine. + + +Requirements +----------------- + +I test on Ruby 1.9.3 and JRuby 1.6.5 in 1.9 mode. Other versions/VMs are +untested. + + +Installation +----------------- + + gem install sidekiq + + +Usage +----------------- + +See `sidekiq -h` for usage details. + + +Client +----------------- + +The Sidekiq client can be used to enqueue messages for processing: + + Sidekiq::Client.push('some_queue', :class => SomeWorker, :args => ['bob', 2, foo: 'bar']) + + +How it works +----------------- + +Sidekiq assumes you are running a Rails 3 application with workers in app/workers. Each message has a format like: + + { class: 'SomeWorker', args: ['bob', 2, {foo: 'bar'}] } + +Sidekiq will instantiate a new instance of SomeWorker and call perform +with args splatted: + + class SomeWorker + def perform(name, count, options) + end + end + +This is the main API difference between Resque and Sidekiq: the perform +method is an *instance* method, not a *class* method. + + +Connections +----------------- + +If your workers are connecting to mongo, memcached, redis, cassandra, +etc you might want to set up a shared connection pool that all workers +can use so you aren't opening a new connection for every message +processed. Sidekiq contains a connection pool API which you can use in your code to +ensure safe, simple access to shared IO connections. Please see the +[connection\_pool gem](https://github.com/mperham/connection_pool) for more information. +Your worker would do something like this: + + class Worker + REDIS_POOL = ConnectionPool.new(:size => 10, :timeout => 3) { Redis.new } + def perform(args) + REDIS_POOL.with_connection do |redis| + redis.lsize(:foo) + end + end + end + +This ensures that if you have a concurrency setting of 50, you'll still only +have a maximum of 10 connections open to Redis. + + +Error Handling +----------------- + +Sidekiq has built-in support for Airbrake. If a worker raises an +exception, Sidekiq will optionally send that error with the message +context to Airbrake, log the error and then replace the worker with a +fresh worker. Just make sure you have Airbrake configured in your Rails +app. + + +Author +----------------- + +Mike Perham, [@mperham](https://twitter.com/mperham), [http://mikeperham.com](http://mikeperham.com) diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..7a972886 --- /dev/null +++ b/Rakefile @@ -0,0 +1,8 @@ +require 'rake/testtask' +Rake::TestTask.new(:test) do |test| + test.libs << 'test' + test.warning = true + test.pattern = 'test/**/test_*.rb' +end + +task :default => :test diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..4e4618d7 --- /dev/null +++ b/TODO.md @@ -0,0 +1,2 @@ +- monitoring hooks +- resque-ui web ui? diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index 76d6b73c..8b8a2ad6 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -26,7 +26,7 @@ module Sidekiq private def enable_rails3 - APP_PATH = File.expand_path('config/application.rb') + #APP_PATH = File.expand_path('config/application.rb') require File.expand_path('config/boot.rb') end @@ -47,10 +47,12 @@ module Sidekiq def parse_options(argv=ARGV) @options = { + :daemon => false, :verbose => false, :queues => [], :worker_count => 25, - :server => 'localhost:6379' + :server => 'localhost:6379', + :pidfile => nil, } @parser = OptionParser.new do |o| @@ -61,6 +63,10 @@ module Sidekiq end end + o.on "-d", "Daemonize" do |arg| + @options[:daemon] = arg + end + o.on "--pidfile PATH", "Use PATH as a pidfile" do |arg| @options[:pidfile] = arg end @@ -73,12 +79,12 @@ module Sidekiq @options[:server] = arg end - o.on '-c', '--count INT', "worker count to use" do |arg| + o.on '-c', '--concurrency INT', "Worker threads to use" do |arg| @options[:worker_count] = arg.to_i end end - @parser.banner = "sidekiq -q foo,1 -q bar,2 " + @parser.banner = "sidekiq -q foo,1 -q bar,2 " @parser.on_tail "-h", "--help", "Show help" do log @parser exit 1 diff --git a/lib/sidekiq/client.rb b/lib/sidekiq/client.rb new file mode 100644 index 00000000..d9066685 --- /dev/null +++ b/lib/sidekiq/client.rb @@ -0,0 +1,41 @@ +require 'multi_json' + +module Sidekiq + class Client + + def self.redis + @redis ||= Redis.new + end + + def self.redis=(redis) + @redis = redis + end + + # Example usage: + # Sidekiq::Client.push('my_queue', :class => MyWorker, :args => ['foo', 1, :bat => 'bar']) + def self.push(queue, item) + raise(ArgumentError, "Message must be a Hash of the form: { :class => SomeClass, :args => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) + raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item[:class] || !item[:args] + + item[:class] = item[:class].to_s if !item[:class].is_a?(String) + redis.rpush("queue:#{queue}", MultiJson.encode(item)) + end + + # Please use .push if possible instead. + # + # Example usage: + # + # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar') + # + # where MyWorker has defined: + # + # def self.queue + # 'my_queue' + # end + # + def self.enqueue(klass, *args) + queue = klass.instance_variable_get(:@queue) || (klass.respond_to?(:queue) && klass.queue) || raise(ArgumentError, "Cannot determine queue to use") + push(queue, { 'class' => klass.name, 'args' => args }) + end + end +end diff --git a/lib/sidekiq/worker.rb b/lib/sidekiq/worker.rb index 08e3adf7..0609416d 100644 --- a/lib/sidekiq/worker.rb +++ b/lib/sidekiq/worker.rb @@ -2,9 +2,20 @@ module Sidekiq class Worker include Celluloid - def process(json) - klass = json['class'].constantize - klass.new.perform(*json['args']) + def process(hash) + begin + klass = hash['class'].constantize + klass.new.perform(*hash['args']) + rescue => ex + airbrake(json, ex) if defined?(::Airbrake) + raise ex + end + end + + def airbrake(json, ex) + ::Airbrake.notify(:error_class => ex.class.name, + :error_message => "#{ex.class.name}: #{e.message}", + :parameters => json) end end end diff --git a/sidekiq.gemspec b/sidekiq.gemspec index 08d7a448..a9c67232 100644 --- a/sidekiq.gemspec +++ b/sidekiq.gemspec @@ -13,8 +13,10 @@ Gem::Specification.new do |gem| gem.name = "sidekiq" gem.require_paths = ["lib"] gem.version = Sidekiq::VERSION + gem.add_dependency 'redis' gem.add_dependency 'connection_pool' gem.add_dependency 'celluloid' + gem.add_dependency 'multi_json' gem.add_development_dependency 'minitest' gem.add_development_dependency 'rake' end diff --git a/test/helper.rb b/test/helper.rb new file mode 100644 index 00000000..cce70235 --- /dev/null +++ b/test/helper.rb @@ -0,0 +1,3 @@ +require 'minitest/unit' +require 'minitest/pride' +require 'minitest/autorun' diff --git a/test/test_client.rb b/test/test_client.rb new file mode 100644 index 00000000..14652014 --- /dev/null +++ b/test/test_client.rb @@ -0,0 +1,17 @@ +require 'helper' +require 'sidekiq/client' + +class TestClient < MiniTest::Unit::TestCase + def test_argument_handling + assert_raises ArgumentError do + Sidekiq::Client.push('foo', 1) + end + + assert_raises ArgumentError do + Sidekiq::Client.push('foo', :class => 'Foo', :noargs => [1, 2]) + end + + count = Sidekiq::Client.push('foo', :class => 'Foo', :args => [1, 2]) + assert count > 0 + end +end