2017-07-19 04:54:39 -04:00
|
|
|
require 'prometheus/client'
|
|
|
|
|
2019-07-02 03:36:33 -04:00
|
|
|
# Keep separate directories for separate processes
|
|
|
|
def prometheus_default_multiproc_dir
|
|
|
|
return unless Rails.env.development? || Rails.env.test?
|
|
|
|
|
2019-12-22 04:07:51 -05:00
|
|
|
if Gitlab::Runtime.sidekiq?
|
2019-07-02 03:36:33 -04:00
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
|
2019-12-22 04:07:51 -05:00
|
|
|
elsif Gitlab::Runtime.unicorn?
|
2019-07-02 03:36:33 -04:00
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir/unicorn')
|
2019-12-22 04:07:51 -05:00
|
|
|
elsif Gitlab::Runtime.puma?
|
2019-07-02 03:36:33 -04:00
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
|
|
|
|
else
|
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-07-19 04:54:39 -04:00
|
|
|
Prometheus::Client.configure do |config|
|
2019-07-10 15:26:47 -04:00
|
|
|
config.logger = Rails.logger # rubocop:disable Gitlab/RailsLogger
|
2017-07-19 04:54:39 -04:00
|
|
|
|
|
|
|
config.initial_mmap_file_size = 4 * 1024
|
|
|
|
|
2019-07-02 03:36:33 -04:00
|
|
|
config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir
|
2017-08-29 09:45:19 -04:00
|
|
|
|
2019-07-18 09:54:11 -04:00
|
|
|
config.pid_provider = Prometheus::PidProvider.method(:worker_id)
|
2017-07-19 04:54:39 -04:00
|
|
|
end
|
2017-08-07 13:13:02 -04:00
|
|
|
|
2017-11-22 18:26:50 -05:00
|
|
|
Gitlab::Application.configure do |config|
|
|
|
|
# 0 should be Sentry to catch errors in this middleware
|
|
|
|
config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
|
|
|
|
end
|
|
|
|
|
2019-06-13 21:41:29 -04:00
|
|
|
Sidekiq.configure_server do |config|
|
|
|
|
config.on(:startup) do
|
2019-12-16 10:07:39 -05:00
|
|
|
# Do not clean the metrics directory here - the supervisor script should
|
|
|
|
# have already taken care of that
|
2019-10-04 02:06:05 -04:00
|
|
|
Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
|
2019-06-13 21:41:29 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-05-24 09:02:39 -04:00
|
|
|
if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
|
2018-10-20 14:00:19 -04:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_worker_start do
|
|
|
|
defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change
|
|
|
|
|
|
|
|
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
|
2020-04-21 11:21:10 -04:00
|
|
|
|
|
|
|
if Gitlab.ee? && Gitlab::Runtime.sidekiq?
|
|
|
|
Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(Settings.monitoring.global_search_sampler_interval).start
|
|
|
|
end
|
2020-01-09 07:08:03 -05:00
|
|
|
rescue IOError => e
|
|
|
|
Gitlab::ErrorTracking.track_exception(e)
|
|
|
|
Gitlab::Metrics.error_detected!
|
2017-11-02 16:23:23 -04:00
|
|
|
end
|
2019-05-29 08:27:44 -04:00
|
|
|
|
2019-07-02 13:05:56 -04:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_master_start do
|
2019-09-17 05:06:11 -04:00
|
|
|
::Prometheus::Client.reinitialize_on_pid_change(force: true)
|
|
|
|
|
2019-12-22 04:07:51 -05:00
|
|
|
if Gitlab::Runtime.unicorn?
|
2019-09-17 05:06:11 -04:00
|
|
|
Gitlab::Metrics::Samplers::UnicornSampler.instance(Settings.monitoring.unicorn_sampler_interval).start
|
2019-12-22 04:07:51 -05:00
|
|
|
elsif Gitlab::Runtime.puma?
|
2019-09-17 05:06:11 -04:00
|
|
|
Gitlab::Metrics::Samplers::PumaSampler.instance(Settings.monitoring.puma_sampler_interval).start
|
2019-05-29 08:27:44 -04:00
|
|
|
end
|
2019-10-02 17:06:22 -04:00
|
|
|
|
|
|
|
Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
|
2020-01-09 07:08:03 -05:00
|
|
|
rescue IOError => e
|
|
|
|
Gitlab::ErrorTracking.track_exception(e)
|
|
|
|
Gitlab::Metrics.error_detected!
|
2019-05-29 08:27:44 -04:00
|
|
|
end
|
2018-10-20 14:00:19 -04:00
|
|
|
end
|
2019-10-04 02:06:05 -04:00
|
|
|
|
2019-12-22 04:07:51 -05:00
|
|
|
if Gitlab::Runtime.web_server?
|
2019-10-04 02:06:05 -04:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_master_start do
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.start
|
|
|
|
end
|
|
|
|
|
2019-11-07 10:06:33 -05:00
|
|
|
# DEPRECATED: TO BE REMOVED
|
|
|
|
# This is needed to implement blackout period of `web_exporter`
|
|
|
|
# https://gitlab.com/gitlab-org/gitlab/issues/35343#note_238479057
|
|
|
|
Gitlab::Cluster::LifecycleEvents.on_before_blackout_period do
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.mark_as_not_running!
|
|
|
|
end
|
|
|
|
|
2019-10-30 05:27:58 -04:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
|
|
|
|
# We need to ensure that before we re-exec or shutdown server
|
2019-10-16 11:06:17 -04:00
|
|
|
# we do stop the exporter
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
|
|
|
|
2019-10-04 02:06:05 -04:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
|
|
|
|
# We need to ensure that before we re-exec server
|
|
|
|
# we do stop the exporter
|
2019-10-16 11:06:17 -04:00
|
|
|
#
|
|
|
|
# We do it again, for being extra safe,
|
|
|
|
# but it should not be needed
|
2019-10-04 02:06:05 -04:00
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
|
|
|
|
|
|
|
Gitlab::Cluster::LifecycleEvents.on_worker_start do
|
|
|
|
# The `#close_on_exec=` takes effect only on `execve`
|
|
|
|
# but this does not happen for Ruby fork
|
|
|
|
#
|
|
|
|
# This does stop server, as it is running on master.
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
|
|
|
end
|