From e9d4b08dc2cde4477e77f5ad4b2e9efad47fbf71 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 11 Jun 2018 21:45:11 -0700 Subject: [PATCH] Clean up stale Prometheus metrics in before_exec unicorn hook Issuing a SIGUSR2 to a unicorn involves a dance where new workers are spun up before the old ones are shut down. To ensure Prometheus .db files aren't deleted after the new workers are shut down, we need to clean these files up before the unicorn process is run. Relates to https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2582 --- config/unicorn.rb.example | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example index cc10da2bd88..220a0191160 100644 --- a/config/unicorn.rb.example +++ b/config/unicorn.rb.example @@ -81,6 +81,17 @@ GC.respond_to?(:copy_on_write_friendly=) and # fast LAN. check_client_connection false +before_exec do |server| + # The following is necessary to ensure stale Prometheus metrics don't + # accumulate over time. It needs to be done in this hook as opposed to + # inside an init script to ensure metrics files aren't deleted after new + # unicorn workers start after a SIGUSR2 is received. + if ENV['prometheus_multiproc_dir'] + old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')] + FileUtils.rm_rf(old_metrics) + end +end + before_fork do |server, worker| # the following is highly recommended for Rails + "preload_app true" # as there's no need for the master process to hold a connection