gitlab-org--gitlab-foss/lib/gitlab/sidekiq_middleware/memory_killer.rb

55 lines
2 KiB
Ruby
Raw Normal View History

module Gitlab
module SidekiqMiddleware
class MemoryKiller
2014-12-08 07:19:31 -05:00
# Default the RSS limit to 0, meaning the MemoryKiller is disabled
MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
# Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
2014-12-08 07:19:31 -05:00
GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
# Wait 30 seconds for running jobs to finish during graceful shutdown
2014-12-08 07:19:31 -05:00
SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGKILL').to_s
2014-12-08 07:19:31 -05:00
# Create a mutex used to ensure there will be only one thread waiting to
# shut Sidekiq down
MUTEX = Mutex.new
def call(worker, job, queue)
yield
current_rss = get_rss
2014-12-08 07:19:31 -05:00
return unless MAX_RSS > 0 && current_rss > MAX_RSS
2014-12-08 07:39:18 -05:00
Thread.new do
# Return if another thread is already waiting to shut Sidekiq down
return unless MUTEX.try_lock
Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
2014-12-08 07:19:31 -05:00
"#{MAX_RSS}"
Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\
"in #{GRACE_TIME} seconds"
2014-12-08 07:19:31 -05:00
sleep(GRACE_TIME)
2016-07-06 04:29:31 -04:00
Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"
Process.kill('SIGTERM', Process.pid)
2014-12-08 07:19:31 -05:00
Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
2016-07-06 04:29:31 -04:00
"#{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"
2014-12-08 07:19:31 -05:00
sleep(SHUTDOWN_WAIT)
Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"
Process.kill(SHUTDOWN_SIGNAL, Process.pid)
end
end
private
def get_rss
output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid}))
return 0 unless status.zero?
output.to_i
end
end
end
end