2014-11-28 14:01:41 +00:00
module Gitlab
module SidekiqMiddleware
class MemoryKiller
2014-12-08 12:19:31 +00:00
# Default the RSS limit to 0, meaning the MemoryKiller is disabled
MAX_RSS = ( ENV [ 'SIDEKIQ_MEMORY_KILLER_MAX_RSS' ] || 0 ) . to_s . to_i
2014-12-05 16:29:34 +00:00
# Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
2014-12-08 12:19:31 +00:00
GRACE_TIME = ( ENV [ 'SIDEKIQ_MEMORY_KILLER_GRACE_TIME' ] || 15 * 60 ) . to_s . to_i
2014-11-28 14:01:41 +00:00
# Wait 30 seconds for running jobs to finish during graceful shutdown
2014-12-08 12:19:31 +00:00
SHUTDOWN_WAIT = ( ENV [ 'SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT' ] || 30 ) . to_s . to_i
# Create a mutex used to ensure there will be only one thread waiting to
# shut Sidekiq down
2014-12-05 16:29:34 +00:00
MUTEX = Mutex . new
2014-11-28 14:01:41 +00:00
def call ( worker , job , queue )
yield
2017-10-31 09:33:30 +00:00
2014-11-28 14:01:41 +00:00
current_rss = get_rss
2014-12-05 16:29:34 +00:00
2014-12-08 12:19:31 +00:00
return unless MAX_RSS > 0 && current_rss > MAX_RSS
2014-11-28 14:01:41 +00:00
2014-12-08 12:39:18 +00:00
Thread . new do
2014-12-05 16:29:34 +00:00
# Return if another thread is already waiting to shut Sidekiq down
return unless MUTEX . try_lock
2017-10-31 09:33:30 +00:00
Sidekiq . logger . warn " Sidekiq worker PID- #{ pid } current RSS #{ current_rss } " \
" exceeds maximum RSS #{ MAX_RSS } after finishing job #{ worker . class } JID- #{ job [ 'jid' ] } "
Sidekiq . logger . warn " Sidekiq worker PID- #{ pid } will stop fetching new jobs in #{ GRACE_TIME } seconds, and will be shut down #{ SHUTDOWN_WAIT } seconds later "
2014-12-05 16:29:34 +00:00
2017-10-31 09:33:30 +00:00
# Wait `GRACE_TIME` to give the memory intensive job time to finish.
# Then, tell Sidekiq to stop fetching new jobs.
wait_and_signal ( GRACE_TIME , 'SIGSTP' , 'stop fetching new jobs' )
2014-12-05 16:29:34 +00:00
2017-10-31 09:33:30 +00:00
# Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
# Then, tell Sidekiq to gracefully shut down by giving jobs a few more
# moments to finish, killing and requeuing them if they didn't, and
# then terminating itself.
wait_and_signal ( SHUTDOWN_WAIT , 'SIGTERM' , 'gracefully shut down' )
2014-12-05 16:29:34 +00:00
2017-10-31 09:33:30 +00:00
# Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
wait_and_signal ( Sidekiq . options [ :timeout ] + 2 , 'SIGKILL' , 'die' )
2014-11-28 14:01:41 +00:00
end
end
private
def get_rss
2017-10-31 09:33:30 +00:00
output , status = Gitlab :: Popen . popen ( %W( ps -o rss= -p #{ pid } ) )
2014-11-28 14:01:41 +00:00
return 0 unless status . zero?
output . to_i
end
2017-10-31 09:33:30 +00:00
def wait_and_signal ( time , signal , explanation )
Sidekiq . logger . warn " waiting #{ time } seconds before sending Sidekiq worker PID- #{ pid } #{ signal } ( #{ explanation } ) "
sleep ( time )
Sidekiq . logger . warn " sending Sidekiq worker PID- #{ pid } #{ signal } ( #{ explanation } ) "
Process . kill ( signal , pid )
end
def pid
Process . pid
end
2014-11-28 14:01:41 +00:00
end
end
end