Logging sidekiq worker class name in SidekiqMemoryKiller

Currently, SidekiqMemoryKiller does not feed worker class name in the
json structured logging. This commit extends the json parameter.
This commit is contained in:
Shinya Maeda 2019-07-22 15:50:50 +07:00
parent 0ec40a68b7
commit ad7acfaad0
2 changed files with 19 additions and 7 deletions

View file

@ -14,9 +14,12 @@ module Gitlab
# shut Sidekiq down # shut Sidekiq down
MUTEX = Mutex.new MUTEX = Mutex.new
attr_reader :worker
def call(worker, job, queue) def call(worker, job, queue)
yield yield
@worker = worker
current_rss = get_rss current_rss = get_rss
return unless MAX_RSS > 0 && current_rss > MAX_RSS return unless MAX_RSS > 0 && current_rss > MAX_RSS
@ -25,9 +28,11 @@ module Gitlab
# Return if another thread is already waiting to shut Sidekiq down # Return if another thread is already waiting to shut Sidekiq down
next unless MUTEX.try_lock next unless MUTEX.try_lock
Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\
" exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}")
Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\
" in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later")
# Wait `GRACE_TIME` to give the memory intensive job time to finish. # Wait `GRACE_TIME` to give the memory intensive job time to finish.
# Then, tell Sidekiq to stop fetching new jobs. # Then, tell Sidekiq to stop fetching new jobs.
@ -59,24 +64,28 @@ module Gitlab
def wait_and_signal_pgroup(time, signal, explanation) def wait_and_signal_pgroup(time, signal, explanation)
return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid
Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal)
sleep(time) sleep(time)
Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal)
Process.kill(signal, 0) Process.kill(signal, 0)
end end
def wait_and_signal(time, signal, explanation) def wait_and_signal(time, signal, explanation)
Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal)
sleep(time) sleep(time)
Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal)
Process.kill(signal, pid) Process.kill(signal, pid)
end end
def pid def pid
Process.pid Process.pid
end end
def warn(message, signal: nil)
Sidekiq.logger.warn(class: worker.class, pid: pid, signal: signal, message: message)
end
end end
end end
end end

View file

@ -45,6 +45,9 @@ describe Gitlab::SidekiqMiddleware::MemoryKiller do
expect(subject).to receive(:sleep).with(10).ordered expect(subject).to receive(:sleep).with(10).ordered
expect(Process).to receive(:kill).with('SIGKILL', pid).ordered expect(Process).to receive(:kill).with('SIGKILL', pid).ordered
expect(Sidekiq.logger)
.to receive(:warn).with(class: 'TestWorker', message: anything, pid: pid, signal: anything).at_least(:once)
run run
end end