2012-01-22 19:01:46 -05:00
|
|
|
require 'celluloid'
|
|
|
|
require 'multi_json'
|
|
|
|
|
2012-01-23 17:05:03 -05:00
|
|
|
require 'sidekiq/util'
|
2012-01-25 16:32:51 -05:00
|
|
|
require 'sidekiq/processor'
|
2012-03-24 16:28:18 -04:00
|
|
|
require 'sidekiq/fetch'
|
2012-01-22 19:01:46 -05:00
|
|
|
|
2012-01-16 19:14:47 -05:00
|
|
|
module Sidekiq
|
|
|
|
|
|
|
|
##
|
2012-02-03 13:02:57 -05:00
|
|
|
# The main router in the system. This
|
2012-03-24 16:28:18 -04:00
|
|
|
# manages the processor state and accepts messages
|
2012-02-03 13:02:57 -05:00
|
|
|
# from Redis to be dispatched to an idle processor.
|
2012-01-16 19:14:47 -05:00
|
|
|
#
|
2012-02-03 13:02:57 -05:00
|
|
|
class Manager
|
2012-01-22 14:32:38 -05:00
|
|
|
include Util
|
2012-01-16 23:02:58 -05:00
|
|
|
include Celluloid
|
2012-01-16 19:18:36 -05:00
|
|
|
|
2012-01-25 16:32:51 -05:00
|
|
|
trap_exit :processor_died
|
2012-01-22 14:32:38 -05:00
|
|
|
|
2012-02-11 16:14:03 -05:00
|
|
|
def initialize(options={})
|
2012-04-21 17:27:31 -04:00
|
|
|
logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
|
2012-03-14 12:56:13 -04:00
|
|
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
2012-02-14 12:00:26 -05:00
|
|
|
logger.debug { options.inspect }
|
2012-02-16 12:45:55 -05:00
|
|
|
@count = options[:concurrency] || 25
|
2012-02-05 16:22:57 -05:00
|
|
|
@done_callback = nil
|
2012-01-22 14:32:38 -05:00
|
|
|
|
2012-04-06 23:53:03 -04:00
|
|
|
@in_progress = {}
|
2012-01-22 19:01:46 -05:00
|
|
|
@done = false
|
|
|
|
@busy = []
|
2012-03-25 22:52:15 -04:00
|
|
|
@fetcher = Fetcher.new(current_actor, options[:queues])
|
2012-02-11 02:19:05 -05:00
|
|
|
@ready = @count.times.map { Processor.new_link(current_actor) }
|
2012-04-17 22:26:56 -04:00
|
|
|
procline
|
2012-01-22 14:32:38 -05:00
|
|
|
end
|
|
|
|
|
2012-03-08 23:58:51 -05:00
|
|
|
def stop(options={})
|
2012-03-25 22:52:15 -04:00
|
|
|
watchdog('Manager#stop died') do
|
|
|
|
shutdown = options[:shutdown]
|
|
|
|
timeout = options[:timeout]
|
2012-03-08 23:58:51 -05:00
|
|
|
|
2012-03-25 22:52:15 -04:00
|
|
|
@done = true
|
2012-05-12 00:25:38 -04:00
|
|
|
Sidekiq::Fetcher.done!
|
2012-04-24 22:58:52 -04:00
|
|
|
@fetcher.terminate! if @fetcher.alive?
|
2012-04-06 23:53:03 -04:00
|
|
|
|
|
|
|
logger.info { "Shutting down #{@ready.size} quiet workers" }
|
2012-03-31 00:22:19 -04:00
|
|
|
@ready.each { |x| x.terminate if x.alive? }
|
2012-03-25 22:52:15 -04:00
|
|
|
@ready.clear
|
2012-01-22 14:32:38 -05:00
|
|
|
|
2012-04-06 23:53:03 -04:00
|
|
|
logger.debug { "Clearing workers in redis" }
|
|
|
|
Sidekiq.redis do |conn|
|
2012-03-25 22:52:15 -04:00
|
|
|
workers = conn.smembers('workers')
|
|
|
|
workers.each do |name|
|
|
|
|
conn.srem('workers', name) if name =~ /:#{process_id}-/
|
|
|
|
end
|
2012-02-11 16:14:03 -05:00
|
|
|
end
|
2012-02-18 00:33:17 -05:00
|
|
|
|
2012-04-06 23:53:03 -04:00
|
|
|
return after(0) { signal(:shutdown) } if @busy.empty?
|
|
|
|
logger.info { "Pausing up to #{timeout} seconds to allow workers to finish..." }
|
2012-04-07 22:33:32 -04:00
|
|
|
hard_shutdown_in timeout if shutdown
|
2012-02-18 00:33:17 -05:00
|
|
|
end
|
2012-01-22 14:32:38 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def start
|
2012-03-25 22:52:15 -04:00
|
|
|
@ready.each { dispatch }
|
2012-01-22 14:32:38 -05:00
|
|
|
end
|
2012-01-16 19:18:36 -05:00
|
|
|
|
2012-02-09 11:15:31 -05:00
|
|
|
def when_done(&blk)
|
|
|
|
@done_callback = blk
|
2012-02-03 13:02:57 -05:00
|
|
|
end
|
|
|
|
|
2012-01-25 16:32:51 -05:00
|
|
|
def processor_done(processor)
|
2012-03-24 16:28:18 -04:00
|
|
|
watchdog('Manager#processor_done died') do
|
2012-02-10 00:46:44 -05:00
|
|
|
@done_callback.call(processor) if @done_callback
|
2012-04-06 23:53:03 -04:00
|
|
|
@in_progress.delete(processor.object_id)
|
2012-02-10 00:46:44 -05:00
|
|
|
@busy.delete(processor)
|
|
|
|
if stopped?
|
2012-02-18 23:01:29 -05:00
|
|
|
processor.terminate if processor.alive?
|
2012-03-11 23:06:20 -04:00
|
|
|
signal(:shutdown) if @busy.empty?
|
2012-02-10 00:46:44 -05:00
|
|
|
else
|
2012-03-12 22:57:04 -04:00
|
|
|
@ready << processor if processor.alive?
|
2012-02-10 00:46:44 -05:00
|
|
|
end
|
|
|
|
dispatch
|
2012-01-16 23:02:58 -05:00
|
|
|
end
|
2012-01-16 19:18:36 -05:00
|
|
|
end
|
2012-01-16 23:02:58 -05:00
|
|
|
|
2012-01-25 16:32:51 -05:00
|
|
|
def processor_died(processor, reason)
|
2012-03-24 16:28:18 -04:00
|
|
|
watchdog("Manager#processor_died died") do
|
2012-04-06 23:53:03 -04:00
|
|
|
@in_progress.delete(processor.object_id)
|
2012-03-24 16:28:18 -04:00
|
|
|
@busy.delete(processor)
|
2012-01-23 17:05:03 -05:00
|
|
|
|
2012-03-24 16:28:18 -04:00
|
|
|
unless stopped?
|
|
|
|
@ready << Processor.new_link(current_actor)
|
|
|
|
dispatch
|
|
|
|
else
|
|
|
|
signal(:shutdown) if @busy.empty?
|
|
|
|
end
|
2012-01-22 19:01:46 -05:00
|
|
|
end
|
2012-01-22 14:32:38 -05:00
|
|
|
end
|
|
|
|
|
2012-03-24 16:28:18 -04:00
|
|
|
def assign(msg, queue)
|
|
|
|
watchdog("Manager#assign died") do
|
2012-04-16 23:18:48 -04:00
|
|
|
if stopped?
|
|
|
|
# Race condition between Manager#stop if Fetcher
|
|
|
|
# is blocked on redis and gets a message after
|
|
|
|
# all the ready Processors have been stopped.
|
|
|
|
# Push the message back to redis.
|
|
|
|
Sidekiq.redis do |conn|
|
|
|
|
conn.lpush("queue:#{queue}", msg)
|
|
|
|
end
|
|
|
|
else
|
|
|
|
processor = @ready.pop
|
|
|
|
@in_progress[processor.object_id] = [msg, queue]
|
|
|
|
@busy << processor
|
2012-04-22 17:02:35 -04:00
|
|
|
processor.process!(Sidekiq.load_json(msg), queue)
|
2012-04-16 23:18:48 -04:00
|
|
|
end
|
2012-01-24 01:08:38 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2012-03-24 16:28:18 -04:00
|
|
|
private
|
2012-01-22 14:32:38 -05:00
|
|
|
|
2012-04-06 23:53:03 -04:00
|
|
|
def hard_shutdown_in(delay)
|
2012-04-07 22:33:32 -04:00
|
|
|
after(delay) do
|
|
|
|
watchdog("Manager#watch_for_shutdown died") do
|
2012-04-06 23:53:03 -04:00
|
|
|
# We've reached the timeout and we still have busy workers.
|
|
|
|
# They must die but their messages shall live on.
|
|
|
|
logger.info("Still waiting for #{@busy.size} busy workers")
|
|
|
|
|
|
|
|
Sidekiq.redis do |conn|
|
|
|
|
@busy.each do |processor|
|
|
|
|
# processor is an actor proxy and we can't call any methods
|
|
|
|
# that would go to the actor (since it's busy). Instead
|
|
|
|
# we'll use the object_id to track the worker's data here.
|
|
|
|
msg, queue = @in_progress[processor.object_id]
|
|
|
|
conn.lpush("queue:#{queue}", msg)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
logger.info("Pushed #{@busy.size} messages back to Redis")
|
|
|
|
|
|
|
|
after(0) { signal(:shutdown) }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2012-03-24 16:28:18 -04:00
|
|
|
def dispatch
|
|
|
|
return if stopped?
|
|
|
|
# This is a safety check to ensure we haven't leaked
|
|
|
|
# processors somehow.
|
|
|
|
raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
|
2012-03-25 22:52:15 -04:00
|
|
|
raise "No ready processor!?" if @ready.empty?
|
2012-03-24 16:28:18 -04:00
|
|
|
|
|
|
|
@fetcher.fetch!
|
2012-01-22 14:32:38 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def stopped?
|
|
|
|
@done
|
|
|
|
end
|
2012-04-17 22:26:56 -04:00
|
|
|
|
|
|
|
def procline
|
2012-04-24 19:32:09 -04:00
|
|
|
$0 = "sidekiq #{Sidekiq::VERSION} [#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
|
2012-04-17 22:26:56 -04:00
|
|
|
after(5) { procline }
|
|
|
|
end
|
2012-01-16 19:14:47 -05:00
|
|
|
end
|
|
|
|
end
|