Merge branch 'sidekiq_memory_killer' into 'master'
Sidekiq memory killer This change adds an optional Sidekiq memory killer middleware that will perform a graceful shutdown when the Sidekiq process is using too much memory. When combined with a process supervision daemon (such as Runit in omnibus-gitlab) this should provide Good Enough handling of the Sidekiq memory leaks we are seeing. The memory killer is off by default. See merge request !1276
This commit is contained in:
commit
27cd35de69
|
@ -4,7 +4,7 @@ v 7.6.0
|
|||
- Add CRON=1 backup setting for quiet backups
|
||||
-
|
||||
-
|
||||
-
|
||||
- Add optional Sidekiq MemoryKiller middleware (enabled via SIDEKIQ_MAX_RSS env variable)
|
||||
-
|
||||
-
|
||||
-
|
||||
|
|
|
@ -15,6 +15,7 @@ Sidekiq.configure_server do |config|
|
|||
|
||||
config.server_middleware do |chain|
|
||||
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger
|
||||
chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MAX_RSS']
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
module Gitlab
|
||||
module SidekiqMiddleware
|
||||
class MemoryKiller
|
||||
# Wait 30 seconds for running jobs to finish during graceful shutdown
|
||||
GRACEFUL_SHUTDOWN_WAIT = 30
|
||||
|
||||
def call(worker, job, queue)
|
||||
yield
|
||||
current_rss = get_rss
|
||||
return unless max_rss > 0 && current_rss > max_rss
|
||||
|
||||
Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
|
||||
"#{max_rss}"
|
||||
Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
|
||||
# SIGUSR1 tells Sidekiq to stop accepting new jobs
|
||||
Process.kill('SIGUSR1', Process.pid)
|
||||
|
||||
Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\
|
||||
"#{Process.pid} in #{graceful_shutdown_wait} seconds"
|
||||
# Send the final shutdown signal to Sidekiq from a separate thread so
|
||||
# that the current job can finish
|
||||
Thread.new do
|
||||
sleep(graceful_shutdown_wait)
|
||||
Process.kill('SIGTERM', Process.pid)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def get_rss
|
||||
output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid}))
|
||||
return 0 unless status.zero?
|
||||
|
||||
output.to_i
|
||||
end
|
||||
|
||||
def max_rss
|
||||
@max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i
|
||||
end
|
||||
|
||||
def graceful_shutdown_wait
|
||||
@graceful_shutdown_wait ||= (
|
||||
ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT
|
||||
).to_i
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue