2018-11-16 19:37:17 -05:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-07-12 18:46:17 -04:00
|
|
|
module Gitlab
|
|
|
|
module Metrics
|
|
|
|
class RequestsRackMiddleware
|
2020-10-28 23:09:25 -04:00
|
|
|
HTTP_METHODS = {
|
|
|
|
"delete" => %w(200 202 204 303 400 401 403 404 500 503),
|
|
|
|
"get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503),
|
|
|
|
"head" => %w(200 204 301 302 303 401 403 404 410 500),
|
|
|
|
"options" => %w(200 404),
|
|
|
|
"patch" => %w(200 202 204 400 403 404 409 416 500),
|
|
|
|
"post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503),
|
|
|
|
"put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500)
|
|
|
|
}.freeze
|
2019-10-02 17:06:22 -04:00
|
|
|
|
2021-07-28 11:09:57 -04:00
|
|
|
HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze
|
2020-04-06 11:10:04 -04:00
|
|
|
|
2021-10-07 11:12:00 -04:00
|
|
|
FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT
|
2021-09-21 14:11:18 -04:00
|
|
|
ENDPOINT_MISSING = 'unknown'
|
2020-10-15 05:08:41 -04:00
|
|
|
|
2020-10-30 11:08:59 -04:00
|
|
|
# These were the top 5 categories at a point in time, chosen as a
|
|
|
|
# reasonable default. If we initialize every category we'll end up
|
|
|
|
# with an explosion in unused metric combinations, but we want the
|
|
|
|
# most common ones to be always present.
|
|
|
|
FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization',
|
|
|
|
'code_review', 'continuous_integration',
|
|
|
|
'not_owned', 'source_code_management',
|
|
|
|
FEATURE_CATEGORY_DEFAULT].freeze
|
|
|
|
|
2017-07-12 18:46:17 -04:00
|
|
|
def initialize(app)
|
|
|
|
@app = app
|
|
|
|
end
|
|
|
|
|
2020-10-28 23:09:25 -04:00
|
|
|
def self.http_requests_total
|
2020-11-13 13:09:11 -05:00
|
|
|
::Gitlab::Metrics.counter(:http_requests_total, 'Request count')
|
2017-07-12 18:46:17 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def self.rack_uncaught_errors_count
|
2020-11-13 13:09:11 -05:00
|
|
|
::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count')
|
2017-07-12 18:46:17 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def self.http_request_duration_seconds
|
2020-11-13 13:09:11 -05:00
|
|
|
::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time',
|
|
|
|
{}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
|
2017-07-12 18:46:17 -04:00
|
|
|
end
|
|
|
|
|
2020-04-06 11:10:04 -04:00
|
|
|
def self.http_health_requests_total
|
2020-11-13 13:09:11 -05:00
|
|
|
::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count')
|
2020-04-06 11:10:04 -04:00
|
|
|
end
|
|
|
|
|
2020-10-28 23:09:25 -04:00
|
|
|
def self.initialize_metrics
|
|
|
|
# This initialization is done to avoid gaps in scraped metrics after
|
|
|
|
# restarts. It makes sure all counters/histograms are available at
|
|
|
|
# process start.
|
|
|
|
#
|
|
|
|
# For example `rate(http_requests_total{status="500"}[1m])` would return
|
|
|
|
# no data until the first 500 error would occur.
|
|
|
|
HTTP_METHODS.each do |method, statuses|
|
2020-10-20 05:08:43 -04:00
|
|
|
http_request_duration_seconds.get({ method: method })
|
2020-10-28 23:09:25 -04:00
|
|
|
|
2020-10-30 11:08:59 -04:00
|
|
|
statuses.product(FEATURE_CATEGORIES_TO_INITIALIZE) do |status, feature_category|
|
2020-10-28 23:09:25 -04:00
|
|
|
http_requests_total.get({ method: method, status: status, feature_category: feature_category })
|
|
|
|
end
|
2019-10-02 17:06:22 -04:00
|
|
|
end
|
2022-02-07 19:14:26 -05:00
|
|
|
|
|
|
|
Gitlab::Metrics::RailsSlis.initialize_request_slis!
|
2019-10-02 17:06:22 -04:00
|
|
|
end
|
|
|
|
|
2017-07-12 18:46:17 -04:00
|
|
|
def call(env)
|
|
|
|
method = env['REQUEST_METHOD'].downcase
|
2020-10-28 23:09:25 -04:00
|
|
|
method = 'INVALID' unless HTTP_METHODS.key?(method)
|
2021-08-13 14:09:11 -04:00
|
|
|
started = ::Gitlab::Metrics::System.monotonic_time
|
2020-09-28 11:09:44 -04:00
|
|
|
health_endpoint = health_endpoint?(env['PATH_INFO'])
|
2020-10-07 14:08:34 -04:00
|
|
|
status = 'undefined'
|
2020-04-06 11:10:04 -04:00
|
|
|
|
2017-07-12 18:46:17 -04:00
|
|
|
begin
|
|
|
|
status, headers, body = @app.call(env)
|
|
|
|
|
2021-08-13 14:09:11 -04:00
|
|
|
elapsed = ::Gitlab::Metrics::System.monotonic_time - started
|
2020-09-28 11:09:44 -04:00
|
|
|
|
2021-08-13 14:09:11 -04:00
|
|
|
if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status)
|
|
|
|
self.class.http_request_duration_seconds.observe({ method: method }, elapsed)
|
2021-09-21 14:11:18 -04:00
|
|
|
|
2021-10-21 23:11:57 -04:00
|
|
|
record_apdex(env, elapsed)
|
2020-09-28 11:09:44 -04:00
|
|
|
end
|
2017-07-12 18:46:17 -04:00
|
|
|
|
|
|
|
[status, headers, body]
|
2021-04-26 08:09:44 -04:00
|
|
|
rescue StandardError
|
2021-08-13 14:09:11 -04:00
|
|
|
self.class.rack_uncaught_errors_count.increment
|
2017-07-12 18:46:17 -04:00
|
|
|
raise
|
2020-10-07 14:08:34 -04:00
|
|
|
ensure
|
|
|
|
if health_endpoint
|
2021-08-13 14:09:11 -04:00
|
|
|
self.class.http_health_requests_total.increment(status: status.to_s, method: method)
|
2020-10-07 14:08:34 -04:00
|
|
|
else
|
2021-08-13 14:09:11 -04:00
|
|
|
self.class.http_requests_total.increment(
|
2020-10-28 23:09:25 -04:00
|
|
|
status: status.to_s,
|
|
|
|
method: method,
|
|
|
|
feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT
|
|
|
|
)
|
2020-10-07 14:08:34 -04:00
|
|
|
end
|
2017-07-12 18:46:17 -04:00
|
|
|
end
|
|
|
|
end
|
2020-04-06 11:10:04 -04:00
|
|
|
|
|
|
|
def health_endpoint?(path)
|
|
|
|
return false if path.blank?
|
|
|
|
|
|
|
|
HEALTH_ENDPOINT.match?(CGI.unescape(path))
|
|
|
|
end
|
2021-06-08 17:10:05 -04:00
|
|
|
|
|
|
|
def feature_category
|
|
|
|
::Gitlab::ApplicationContext.current_context_attribute(:feature_category)
|
|
|
|
end
|
2021-09-21 14:11:18 -04:00
|
|
|
|
|
|
|
def endpoint_id
|
|
|
|
::Gitlab::ApplicationContext.current_context_attribute(:caller_id)
|
|
|
|
end
|
|
|
|
|
2021-10-21 23:11:57 -04:00
|
|
|
def record_apdex(env, elapsed)
|
2021-10-19 08:12:07 -04:00
|
|
|
urgency = urgency_for_env(env)
|
|
|
|
|
2021-09-23 18:34:10 -04:00
|
|
|
Gitlab::Metrics::RailsSlis.request_apdex.increment(
|
2021-10-19 08:12:07 -04:00
|
|
|
labels: labels_from_context.merge(request_urgency: urgency.name),
|
|
|
|
success: elapsed < urgency.duration
|
2021-09-21 14:11:18 -04:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def labels_from_context
|
|
|
|
{
|
|
|
|
feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT,
|
|
|
|
endpoint_id: endpoint_id.presence || ENDPOINT_MISSING
|
|
|
|
}
|
|
|
|
end
|
2021-09-28 08:11:10 -04:00
|
|
|
|
2021-10-19 08:12:07 -04:00
|
|
|
def urgency_for_env(env)
|
|
|
|
endpoint_urgency =
|
2021-09-28 08:11:10 -04:00
|
|
|
if env['api.endpoint'].present?
|
2021-10-05 08:12:58 -04:00
|
|
|
env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint'])
|
|
|
|
elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency)
|
|
|
|
env['action_controller.instance'].urgency
|
2021-09-28 08:11:10 -04:00
|
|
|
end
|
|
|
|
|
2021-10-19 08:12:07 -04:00
|
|
|
endpoint_urgency || Gitlab::EndpointAttributes::DEFAULT_URGENCY
|
2021-09-28 08:11:10 -04:00
|
|
|
end
|
2017-07-12 18:46:17 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|