diff --git a/lib/sidekiq/launcher.rb b/lib/sidekiq/launcher.rb index 9c294a5b..3250c76d 100644 --- a/lib/sidekiq/launcher.rb +++ b/lib/sidekiq/launcher.rb @@ -188,6 +188,10 @@ module Sidekiq end end + # We run the heartbeat every five seconds. + # Capture five samples of RTT, log a warning if each sample + # is above our warning threshold. + RTT_READINGS = RingBuffer.new(5) RTT_WARNING_LEVEL = 50_000 def check_rtt @@ -198,15 +202,17 @@ module Sidekiq b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) end rtt = b - a + RTT_READINGS << rtt # Ideal RTT for Redis is < 1000µs # Workable is < 10,000µs # Log a warning if it's a disaster. - if rtt > RTT_WARNING_LEVEL - Sidekiq.logger.warn <<-EOM + if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL } + Sidekiq.logger.warn <<~EOM Your Redis network connection is performing extremely poorly. - Current RTT is #{rtt} µs, ideally this should be < 1000. + Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000. Ensure Redis is running in the same AZ or datacenter as Sidekiq. EOM + RTT_READINGS.reset end rtt end diff --git a/lib/sidekiq/util.rb b/lib/sidekiq/util.rb index 2346f4c5..5e1294e4 100644 --- a/lib/sidekiq/util.rb +++ b/lib/sidekiq/util.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "forwardable" require "socket" require "securerandom" require "sidekiq/exception_handler" @@ -8,6 +9,33 @@ module Sidekiq ## # This module is part of Sidekiq core and not intended for extensions. # + + class RingBuffer + include Enumerable + extend Forwardable + def_delegators :@buf, :[], :each, :size + + def initialize(size, default=0) + @size = size + @buf = Array.new(size, default) + @index = 0 + end + + def <<(element) + @buf[@index % @size] = element + @index += 1 + element + end + + def buffer + @buf + end + + def reset(default=0) + @buf.fill(default) + end + end + module Util include ExceptionHandler