diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index df8968cf677..f5c65e75af0 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -1,36 +1,39 @@ module Gitlab module SidekiqMiddleware class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = 15 * 60 + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = 30 - # Create a mutex so that there will be only one thread waiting to shut - # Sidekiq down + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down MUTEX = Mutex.new def call(worker, job, queue) yield current_rss = get_rss - return unless max_rss > 0 && current_rss > max_rss + return unless MAX_RSS > 0 && current_rss > MAX_RSS Tread.new do # Return if another thread is already waiting to shut Sidekiq down return unless MUTEX.try_lock Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{max_rss}" + "#{MAX_RSS}" Sidekiq.logger.warn "spawned thread that will shut down PID "\ - "#{Process.pid} in #{grace_time} seconds" - sleep(grace_time) + "#{Process.pid} in #{GRACE_TIME} seconds" + sleep(GRACE_TIME) Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" Process.kill('SIGUSR1', Process.pid) - Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\ + Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ "SIGTERM to PID #{Process.pid}" - sleep(shutdown_wait) + sleep(SHUTDOWN_WAIT) Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" Process.kill('SIGTERM', Process.pid) @@ -45,22 +48,6 @@ module Gitlab output.to_i end - - def max_rss - @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i - end - - def shutdown_wait - @graceful_shutdown_wait ||= ( - ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT - ).to_i - end - - def grace_time - @grace_time ||= ( - ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME - ).to_i - end end end end