From 64ab6c9ed54d1c0a86f4c3bb6b87fcac882da0c0 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 28 Nov 2014 15:01:41 +0100 Subject: [PATCH 1/2] Add 'MemoryKiller' Sidekiq middleware When enabled, this middleware allows Sidekiq to detect that its RSS has exceeded a maximum value, triggering a graceful shutdown. This middleware should be combined with external process supervision that will restart Sidekiq after the graceful shutdown, such as Runit. --- CHANGELOG | 2 +- config/initializers/4_sidekiq.rb | 1 + .../sidekiq_middleware/memory_killer.rb | 45 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/sidekiq_middleware/memory_killer.rb diff --git a/CHANGELOG b/CHANGELOG index 417bd3c2b4d..85178d0dfd7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,7 +4,7 @@ v 7.6.0 - Add CRON=1 backup setting for quiet backups - - - - + - Add optional Sidekiq MemoryKiller middleware (enabled via SIDEKIQ_MAX_RSS env variable) - - - diff --git a/config/initializers/4_sidekiq.rb b/config/initializers/4_sidekiq.rb index 228b14cb526..b8a7fd624a5 100644 --- a/config/initializers/4_sidekiq.rb +++ b/config/initializers/4_sidekiq.rb @@ -15,6 +15,7 @@ Sidekiq.configure_server do |config| config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger + chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MAX_RSS'] end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..3ef46627916 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,45 @@ +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Wait 30 seconds for running jobs to finish during graceful shutdown + GRACEFUL_SHUTDOWN_WAIT = 30 + + def call(worker, job, queue) + yield + current_rss = get_rss + return unless max_rss > 0 && current_rss > max_rss + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{max_rss}" + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ + "#{Process.pid} in #{graceful_shutdown_wait} seconds" + Thread.new do + sleep(graceful_shutdown_wait) + Process.kill('SIGTERM', Process.pid) + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid})) + return 0 unless status.zero? + + output.to_i + end + + def max_rss + @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i + end + + def graceful_shutdown_wait + @graceful_shutdown_wait ||= ( + ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT + ).to_i + end + end + end +end From d336127a20ce22a9512123595a887c4207b748e9 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 28 Nov 2014 15:19:03 +0100 Subject: [PATCH 2/2] Add comments to the MemoryKiller middleware --- lib/gitlab/sidekiq_middleware/memory_killer.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 3ef46627916..0fb09d3f228 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -12,10 +12,13 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{max_rss}" Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + # SIGUSR1 tells Sidekiq to stop accepting new jobs Process.kill('SIGUSR1', Process.pid) Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ "#{Process.pid} in #{graceful_shutdown_wait} seconds" + # Send the final shutdown signal to Sidekiq from a separate thread so + # that the current job can finish Thread.new do sleep(graceful_shutdown_wait) Process.kill('SIGTERM', Process.pid)