diff --git a/Changes.md b/Changes.md index 470a9535..9dfe0658 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,7 @@ HEAD --------- +- Store Redis RTT and log if poor [#4824] - Add process/thread stats to Busy page [#4806] - Refactor Web UI session usage. [#4804] Numerous people have hit "Forbidden" errors and struggled with Sidekiq's diff --git a/lib/sidekiq/api.rb b/lib/sidekiq/api.rb index 05ab5f5b..7c483bf7 100644 --- a/lib/sidekiq/api.rb +++ b/lib/sidekiq/api.rb @@ -791,12 +791,12 @@ module Sidekiq # you'll be happier this way conn.pipelined do procs.each do |key| - conn.hmget(key, "info", "busy", "beat", "quiet", "rss") + conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us") end end } - result.each do |info, busy, at_s, quiet, rss| + result.each do |info, busy, at_s, quiet, rss, rtt| # If a process is stopped between when we query Redis for `procs` and # when we query for `result`, we will have an item in `result` that is # composed of `nil` values. @@ -806,7 +806,8 @@ module Sidekiq yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet, - "rss" => rss.to_i)) + "rss" => rss.to_i, + "rtt_us" => rtt.to_i)) end end diff --git a/lib/sidekiq/launcher.rb b/lib/sidekiq/launcher.rb index 95854bbd..9c294a5b 100644 --- a/lib/sidekiq/launcher.rb +++ b/lib/sidekiq/launcher.rb @@ -153,6 +153,8 @@ module Sidekiq end end + rtt = check_rtt + fails = procd = 0 kb = memory_usage(::Process.pid) @@ -163,6 +165,7 @@ module Sidekiq conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, + "rtt_us", rtt, "quiet", @done, "rss", kb) conn.expire(key, 60) @@ -185,6 +188,29 @@ module Sidekiq end end + RTT_WARNING_LEVEL = 50_000 + + def check_rtt + a = b = 0 + Sidekiq.redis do |x| + a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) + x.ping + b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) + end + rtt = b - a + # Ideal RTT for Redis is < 1000µs + # Workable is < 10,000µs + # Log a warning if it's a disaster. + if rtt > RTT_WARNING_LEVEL + Sidekiq.logger.warn <<-EOM + Your Redis network connection is performing extremely poorly. + Current RTT is #{rtt} µs, ideally this should be < 1000. + Ensure Redis is running in the same AZ or datacenter as Sidekiq. + EOM + end + rtt + end + MEMORY_GRABBER = case RUBY_PLATFORM when /linux/ ->(pid) { diff --git a/test/test_launcher.rb b/test/test_launcher.rb index 4d0c740d..39f1de0d 100644 --- a/test/test_launcher.rb +++ b/test/test_launcher.rb @@ -49,9 +49,11 @@ describe Sidekiq::Launcher do it 'stores process info in redis' do subject.heartbeat - workers = Sidekiq.redis { |c| c.hmget(subject.identity, 'busy') } + workers, rtt = Sidekiq.redis { |c| c.hmget(subject.identity, 'busy', 'rtt_us') } - assert_equal ["1"], workers + assert_equal "1", workers + refute_nil rtt + assert_in_delta 1000, rtt.to_i, 1000 expires = Sidekiq.redis { |c| c.pttl(subject.identity) }