Merge pull request #101 from 907th/fix_redis_reloading_bug
Fix stuck queues bug on Redis restart (fix #75 and #72)
This commit is contained in:
commit
ee2ee08e0a
|
@ -14,6 +14,8 @@ module Sidekiq::LimitFetch
|
||||||
require_relative 'extensions/queue'
|
require_relative 'extensions/queue'
|
||||||
require_relative 'extensions/manager'
|
require_relative 'extensions/manager'
|
||||||
|
|
||||||
|
TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
|
||||||
|
|
||||||
extend self
|
extend self
|
||||||
|
|
||||||
def new(_)
|
def new(_)
|
||||||
|
@ -39,14 +41,21 @@ module Sidekiq::LimitFetch
|
||||||
def redis_retryable
|
def redis_retryable
|
||||||
yield
|
yield
|
||||||
rescue Redis::BaseConnectionError
|
rescue Redis::BaseConnectionError
|
||||||
sleep 1
|
sleep TIMEOUT
|
||||||
retry
|
retry
|
||||||
|
rescue Redis::CommandError => error
|
||||||
|
# If Redis was restarted and is still loading its snapshot,
|
||||||
|
# then we should treat this as a temporary connection error too.
|
||||||
|
if error.message =~ /^LOADING/
|
||||||
|
sleep TIMEOUT
|
||||||
|
retry
|
||||||
|
else
|
||||||
|
raise
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
TIMEOUT = Sidekiq::BasicFetch::TIMEOUT
|
|
||||||
|
|
||||||
def redis_brpop(queues)
|
def redis_brpop(queues)
|
||||||
if queues.empty?
|
if queues.empty?
|
||||||
sleep TIMEOUT # there are no queues to handle, so lets sleep
|
sleep TIMEOUT # there are no queues to handle, so lets sleep
|
||||||
|
|
|
@ -20,15 +20,17 @@ module Sidekiq::LimitFetch::Queues
|
||||||
end
|
end
|
||||||
|
|
||||||
def acquire
|
def acquire
|
||||||
selector.acquire(ordered_queues, namespace)
|
queues = saved
|
||||||
.tap {|it| save it }
|
queues ||= Sidekiq::LimitFetch.redis_retryable do
|
||||||
.map {|it| "queue:#{it}" }
|
selector.acquire(ordered_queues, namespace)
|
||||||
|
end
|
||||||
|
save queues
|
||||||
|
queues.map { |it| "queue:#{it}" }
|
||||||
end
|
end
|
||||||
|
|
||||||
def release_except(full_name)
|
def release_except(full_name)
|
||||||
queues = restore
|
queues = restore
|
||||||
queues.delete full_name[/queue:(.*)/, 1] if full_name
|
queues.delete full_name[/queue:(.*)/, 1] if full_name
|
||||||
|
|
||||||
Sidekiq::LimitFetch.redis_retryable do
|
Sidekiq::LimitFetch.redis_retryable do
|
||||||
selector.release queues, namespace
|
selector.release queues, namespace
|
||||||
end
|
end
|
||||||
|
@ -141,13 +143,17 @@ module Sidekiq::LimitFetch::Queues
|
||||||
Sidekiq::LimitFetch::Global::Selector
|
Sidekiq::LimitFetch::Global::Selector
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def saved
|
||||||
|
Thread.current[THREAD_KEY]
|
||||||
|
end
|
||||||
|
|
||||||
def save(queues)
|
def save(queues)
|
||||||
Thread.current[THREAD_KEY] = queues
|
Thread.current[THREAD_KEY] = queues
|
||||||
end
|
end
|
||||||
|
|
||||||
def restore
|
def restore
|
||||||
Thread.current[THREAD_KEY] || []
|
saved || []
|
||||||
ensure
|
ensure
|
||||||
Thread.current[THREAD_KEY] = nil
|
save nil
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -15,61 +15,72 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
|
||||||
|
|
||||||
before { subject.start options }
|
before { subject.start options }
|
||||||
|
|
||||||
|
def in_thread(&block)
|
||||||
|
thr = Thread.new(&block)
|
||||||
|
thr.join
|
||||||
|
end
|
||||||
|
|
||||||
it 'should acquire queues' do
|
it 'should acquire queues' do
|
||||||
subject.acquire
|
in_thread { subject.acquire }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should acquire dynamically blocking queues' do
|
it 'should acquire dynamically blocking queues' do
|
||||||
subject.acquire
|
in_thread { subject.acquire }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
|
|
||||||
Sidekiq::Queue['queue1'].block
|
Sidekiq::Queue['queue1'].block
|
||||||
|
|
||||||
subject.acquire
|
in_thread { subject.acquire }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 2
|
expect(Sidekiq::Queue['queue1'].probed).to eq 2
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should block except given queues' do
|
it 'should block except given queues' do
|
||||||
Sidekiq::Queue['queue1'].block_except 'queue2'
|
Sidekiq::Queue['queue1'].block_except 'queue2'
|
||||||
subject.acquire
|
in_thread { subject.acquire }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
|
|
||||||
Sidekiq::Queue['queue1'].block_except 'queue404'
|
Sidekiq::Queue['queue1'].block_except 'queue404'
|
||||||
subject.acquire
|
in_thread { subject.acquire }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 2
|
expect(Sidekiq::Queue['queue1'].probed).to eq 2
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should release queues' do
|
it 'should release queues' do
|
||||||
subject.acquire
|
in_thread {
|
||||||
subject.release_except nil
|
subject.acquire
|
||||||
|
subject.release_except nil
|
||||||
|
}
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 0
|
expect(Sidekiq::Queue['queue1'].probed).to eq 0
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should release queues except selected' do
|
it 'should release queues except selected' do
|
||||||
subject.acquire
|
in_thread {
|
||||||
subject.release_except 'queue:queue1'
|
subject.acquire
|
||||||
|
subject.release_except 'queue:queue1'
|
||||||
|
}
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'should release when no queues was acquired' do
|
it 'should release when no queues was acquired' do
|
||||||
queues.each {|name| Sidekiq::Queue[name].pause }
|
queues.each {|name| Sidekiq::Queue[name].pause }
|
||||||
subject.acquire
|
in_thread {
|
||||||
expect { subject.release_except nil }.not_to raise_exception
|
subject.acquire
|
||||||
|
expect { subject.release_except nil }.not_to raise_exception
|
||||||
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'blocking' do
|
context 'blocking' do
|
||||||
let(:blocking) { %w(queue1) }
|
let(:blocking) { %w(queue1) }
|
||||||
|
|
||||||
it 'should acquire blocking queues' do
|
it 'should acquire blocking queues' do
|
||||||
3.times { subject.acquire }
|
3.times { in_thread { subject.acquire } }
|
||||||
expect(Sidekiq::Queue['queue1'].probed).to eq 3
|
expect(Sidekiq::Queue['queue1'].probed).to eq 3
|
||||||
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
||||||
end
|
end
|
||||||
|
|
|
@ -6,7 +6,7 @@ RSpec.describe Sidekiq::LimitFetch do
|
||||||
let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
|
let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
|
||||||
|
|
||||||
before do
|
before do
|
||||||
subject::Queues.start options
|
subject::Queues.start options
|
||||||
|
|
||||||
Sidekiq.redis do |it|
|
Sidekiq.redis do |it|
|
||||||
it.del 'queue:queue1'
|
it.del 'queue:queue1'
|
||||||
|
|
Loading…
Reference in New Issue