From 4f9a14061b707f39d8a98dae328089bbfbc09e70 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 5 Dec 2014 17:29:34 +0100 Subject: [PATCH 1/4] Wait 15 minutes before Sidekiq MemoryKiller action --- .../sidekiq_middleware/memory_killer.rb | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 0fb09d3f228..df8968cf677 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -1,26 +1,38 @@ module Gitlab module SidekiqMiddleware class MemoryKiller + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = 15 * 60 # Wait 30 seconds for running jobs to finish during graceful shutdown - GRACEFUL_SHUTDOWN_WAIT = 30 + SHUTDOWN_WAIT = 30 + # Create a mutex so that there will be only one thread waiting to shut + # Sidekiq down + MUTEX = Mutex.new def call(worker, job, queue) yield current_rss = get_rss + return unless max_rss > 0 && current_rss > max_rss - Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{max_rss}" - Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" - # SIGUSR1 tells Sidekiq to stop accepting new jobs - Process.kill('SIGUSR1', Process.pid) + Tread.new do + # Return if another thread is already waiting to shut Sidekiq down + return unless MUTEX.try_lock - Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ - "#{Process.pid} in #{graceful_shutdown_wait} seconds" - # Send the final shutdown signal to Sidekiq from a separate thread so - # that the current job can finish - Thread.new do - sleep(graceful_shutdown_wait) + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{max_rss}" + Sidekiq.logger.warn "spawned thread that will shut down PID "\ + "#{Process.pid} in #{grace_time} seconds" + sleep(grace_time) + + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\ + "SIGTERM to PID #{Process.pid}" + sleep(shutdown_wait) + + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" Process.kill('SIGTERM', Process.pid) end end @@ -38,9 +50,15 @@ module Gitlab @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i end - def graceful_shutdown_wait + def shutdown_wait @graceful_shutdown_wait ||= ( - ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT + ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT + ).to_i + end + + def grace_time + @grace_time ||= ( + ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME ).to_i end end From 3dd86b83baccc2a2aecc12a1f4a4819438c62a81 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 8 Dec 2014 13:19:31 +0100 Subject: [PATCH 2/4] Use constants instead of getters --- .../sidekiq_middleware/memory_killer.rb | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index df8968cf677..f5c65e75af0 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -1,36 +1,39 @@ module Gitlab module SidekiqMiddleware class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = 15 * 60 + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = 30 - # Create a mutex so that there will be only one thread waiting to shut - # Sidekiq down + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down MUTEX = Mutex.new def call(worker, job, queue) yield current_rss = get_rss - return unless max_rss > 0 && current_rss > max_rss + return unless MAX_RSS > 0 && current_rss > MAX_RSS Tread.new do # Return if another thread is already waiting to shut Sidekiq down return unless MUTEX.try_lock Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{max_rss}" + "#{MAX_RSS}" Sidekiq.logger.warn "spawned thread that will shut down PID "\ - "#{Process.pid} in #{grace_time} seconds" - sleep(grace_time) + "#{Process.pid} in #{GRACE_TIME} seconds" + sleep(GRACE_TIME) Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" Process.kill('SIGUSR1', Process.pid) - Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\ + Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ "SIGTERM to PID #{Process.pid}" - sleep(shutdown_wait) + sleep(SHUTDOWN_WAIT) Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" Process.kill('SIGTERM', Process.pid) @@ -45,22 +48,6 @@ module Gitlab output.to_i end - - def max_rss - @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i - end - - def shutdown_wait - @graceful_shutdown_wait ||= ( - ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT - ).to_i - end - - def grace_time - @grace_time ||= ( - ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME - ).to_i - end end end end From 2fcef3278ce4ccb62cefa590b0f65509028fbcc0 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 8 Dec 2014 13:39:18 +0100 Subject: [PATCH 3/4] Fix typo --- lib/gitlab/sidekiq_middleware/memory_killer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index f5c65e75af0..0f2db50e98c 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -18,7 +18,7 @@ module Gitlab return unless MAX_RSS > 0 && current_rss > MAX_RSS - Tread.new do + Thread.new do # Return if another thread is already waiting to shut Sidekiq down return unless MUTEX.try_lock From b8dfd63eacc26bb2ef6a900b75e794c33cf2c9ca Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 8 Dec 2014 13:39:25 +0100 Subject: [PATCH 4/4] Use the new SIDEKIQ_MEMORY_KILLER_MAX_RSS variable --- config/initializers/4_sidekiq.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/4_sidekiq.rb b/config/initializers/4_sidekiq.rb index 75c543c0f47..e856499732e 100644 --- a/config/initializers/4_sidekiq.rb +++ b/config/initializers/4_sidekiq.rb @@ -15,7 +15,7 @@ Sidekiq.configure_server do |config| config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] - chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MAX_RSS'] + chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] end end