2021-02-03 21:09:17 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2022-03-15 12:07:44 +00:00
|
|
|
PUMA_EXTERNAL_METRICS_SERVER = Gitlab::Utils.to_boolean(ENV['PUMA_EXTERNAL_METRICS_SERVER'])
|
2022-03-17 12:07:26 +00:00
|
|
|
require Rails.root.join('metrics_server', 'metrics_server') if PUMA_EXTERNAL_METRICS_SERVER
|
2022-03-15 12:07:44 +00:00
|
|
|
|
2019-07-02 07:36:33 +00:00
|
|
|
# Keep separate directories for separate processes
|
2022-03-25 12:07:52 +00:00
|
|
|
def metrics_temp_dir
|
2019-07-02 07:36:33 +00:00
|
|
|
return unless Rails.env.development? || Rails.env.test?
|
|
|
|
|
2019-12-22 09:07:51 +00:00
|
|
|
if Gitlab::Runtime.sidekiq?
|
2019-07-02 07:36:33 +00:00
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
|
2019-12-22 09:07:51 +00:00
|
|
|
elsif Gitlab::Runtime.puma?
|
2019-07-02 07:36:33 +00:00
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
|
|
|
|
else
|
|
|
|
Rails.root.join('tmp/prometheus_multiproc_dir')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-03-25 12:07:52 +00:00
|
|
|
def prometheus_metrics_dir
|
|
|
|
ENV['prometheus_multiproc_dir'] || metrics_temp_dir
|
2022-03-09 18:08:16 +00:00
|
|
|
end
|
|
|
|
|
2022-03-25 12:07:52 +00:00
|
|
|
def puma_metrics_server_process?
|
|
|
|
Prometheus::PidProvider.worker_id == 'puma_master'
|
2022-03-09 18:08:16 +00:00
|
|
|
end
|
|
|
|
|
2022-03-25 12:07:52 +00:00
|
|
|
if puma_metrics_server_process?
|
2022-03-09 18:08:16 +00:00
|
|
|
# The following is necessary to ensure stale Prometheus metrics don't accumulate over time.
|
2022-03-25 12:07:52 +00:00
|
|
|
# It needs to be done as early as possible to ensure new metrics aren't being deleted.
|
|
|
|
#
|
|
|
|
# Note that this should not happen for Sidekiq. Since Sidekiq workers are spawned from the
|
|
|
|
# sidekiq-cluster script, we perform this cleanup in `sidekiq_cluster/cli.rb` instead,
|
|
|
|
# since it must happen prior to any worker processes or the metrics server starting up.
|
|
|
|
Prometheus::CleanupMultiprocDirService.new(prometheus_metrics_dir).execute
|
2022-03-09 18:08:16 +00:00
|
|
|
|
|
|
|
::Prometheus::Client.reinitialize_on_pid_change(force: true)
|
|
|
|
end
|
|
|
|
|
2021-06-18 15:10:16 +00:00
|
|
|
::Prometheus::Client.configure do |config|
|
2020-09-09 12:08:22 +00:00
|
|
|
config.logger = Gitlab::AppLogger
|
2017-07-19 08:54:39 +00:00
|
|
|
|
2022-03-25 12:07:52 +00:00
|
|
|
config.multiprocess_files_dir = prometheus_metrics_dir
|
2017-08-29 13:45:19 +00:00
|
|
|
|
2021-06-18 15:10:16 +00:00
|
|
|
config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
|
2017-07-19 08:54:39 +00:00
|
|
|
end
|
2017-08-07 17:13:02 +00:00
|
|
|
|
2017-11-22 23:26:50 +00:00
|
|
|
Gitlab::Application.configure do |config|
|
|
|
|
# 0 should be Sentry to catch errors in this middleware
|
2021-06-08 21:10:05 +00:00
|
|
|
config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware)
|
2017-11-22 23:26:50 +00:00
|
|
|
end
|
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
# Any actions beyond this check should only execute outside of tests, when running in an application
|
|
|
|
# context (i.e. not in the Rails console or rspec) and when users have enabled metrics.
|
|
|
|
return if Rails.env.test? || !Gitlab::Runtime.application? || !Gitlab::Metrics.prometheus_metrics_enabled?
|
|
|
|
|
2021-11-18 12:13:46 +00:00
|
|
|
if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0')
|
|
|
|
# The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0)
|
|
|
|
# in a cluster of processes, is responsible for serving health checks.
|
2021-11-23 15:11:19 +00:00
|
|
|
#
|
|
|
|
# Do not clean the metrics directory here - the supervisor script should
|
|
|
|
# have already taken care of that.
|
2021-11-18 12:13:46 +00:00
|
|
|
Sidekiq.configure_server do |config|
|
|
|
|
config.on(:startup) do
|
2021-11-23 15:11:19 +00:00
|
|
|
# In https://gitlab.com/gitlab-org/gitlab/-/issues/345804 we are looking to
|
|
|
|
# only serve health-checks from a worker process; for backwards compatibility
|
|
|
|
# we still go through the metrics exporter server, but start to configure it
|
|
|
|
# with the new settings keys.
|
|
|
|
exporter_settings = Settings.monitoring.sidekiq_health_checks
|
|
|
|
Gitlab::Metrics::Exporter::SidekiqExporter.instance(exporter_settings).start
|
2021-11-18 12:13:46 +00:00
|
|
|
end
|
2019-06-14 01:41:29 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_master_start do
|
|
|
|
Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
|
2020-09-09 09:08:40 +00:00
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
if Gitlab::Runtime.puma?
|
|
|
|
Gitlab::Metrics::Samplers::PumaSampler.instance.start
|
2021-01-22 21:09:10 +00:00
|
|
|
|
2022-03-17 12:07:26 +00:00
|
|
|
if PUMA_EXTERNAL_METRICS_SERVER && Settings.monitoring.web_exporter.enabled
|
2022-03-15 12:07:44 +00:00
|
|
|
MetricsServer.start_for_puma
|
|
|
|
else
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.start
|
|
|
|
end
|
2017-11-02 20:23:23 +00:00
|
|
|
end
|
2019-05-29 12:27:44 +00:00
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
Gitlab::Ci::Parsers.instrument!
|
|
|
|
rescue IOError => e
|
|
|
|
Gitlab::ErrorTracking.track_exception(e)
|
|
|
|
Gitlab::Metrics.error_detected!
|
|
|
|
end
|
2019-10-02 21:06:22 +00:00
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_worker_start do
|
|
|
|
defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change
|
|
|
|
logger = Gitlab::AppLogger
|
|
|
|
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(logger: logger).start
|
|
|
|
Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
|
|
|
|
Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start
|
|
|
|
|
|
|
|
if Gitlab::Runtime.puma?
|
2022-03-15 12:07:44 +00:00
|
|
|
# Since we are observing a metrics server from the Puma primary, we would inherit
|
|
|
|
# this supervision thread after forking into workers, so we need to explicitly stop it here.
|
|
|
|
if PUMA_EXTERNAL_METRICS_SERVER
|
|
|
|
::MetricsServer::PumaProcessSupervisor.instance.stop
|
|
|
|
else
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
2020-05-15 21:08:21 +00:00
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
|
|
|
|
end
|
2021-01-22 21:09:10 +00:00
|
|
|
|
2022-03-07 15:22:51 +00:00
|
|
|
if Gitlab.ee? && Gitlab::Runtime.sidekiq?
|
|
|
|
Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start
|
2019-05-29 12:27:44 +00:00
|
|
|
end
|
2022-03-07 15:22:51 +00:00
|
|
|
|
|
|
|
Gitlab::Ci::Parsers.instrument!
|
|
|
|
rescue IOError => e
|
|
|
|
Gitlab::ErrorTracking.track_exception(e)
|
|
|
|
Gitlab::Metrics.error_detected!
|
2018-10-20 18:00:19 +00:00
|
|
|
end
|
2019-10-04 06:06:05 +00:00
|
|
|
|
2022-03-02 15:16:07 +00:00
|
|
|
if Gitlab::Runtime.puma?
|
2019-10-30 09:27:58 +00:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
|
|
|
|
# We need to ensure that before we re-exec or shutdown server
|
2022-03-15 12:07:44 +00:00
|
|
|
# we also stop the metrics server
|
|
|
|
if PUMA_EXTERNAL_METRICS_SERVER
|
|
|
|
::MetricsServer::PumaProcessSupervisor.instance.shutdown
|
|
|
|
else
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
2019-10-16 15:06:17 +00:00
|
|
|
end
|
|
|
|
|
2019-10-04 06:06:05 +00:00
|
|
|
Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
|
|
|
|
# We need to ensure that before we re-exec server
|
2022-03-15 12:07:44 +00:00
|
|
|
# we also stop the metrics server
|
2019-10-16 15:06:17 +00:00
|
|
|
#
|
|
|
|
# We do it again, for being extra safe,
|
|
|
|
# but it should not be needed
|
2022-03-15 12:07:44 +00:00
|
|
|
if PUMA_EXTERNAL_METRICS_SERVER
|
|
|
|
::MetricsServer::PumaProcessSupervisor.instance.shutdown
|
|
|
|
else
|
|
|
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
|
|
|
end
|
2019-10-04 06:06:05 +00:00
|
|
|
end
|
|
|
|
end
|