2018-10-22 03:00:50 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-03-07 11:57:42 -05:00
|
|
|
module Gitlab
|
|
|
|
# Helper methods to interact with Prometheus network services & resources
|
2017-04-26 16:09:03 -04:00
|
|
|
class PrometheusClient
|
2019-08-06 22:42:20 -04:00
|
|
|
include Gitlab::Utils::StrongMemoize
|
2018-02-23 12:58:40 -05:00
|
|
|
Error = Class.new(StandardError)
|
2020-07-03 11:09:13 -04:00
|
|
|
ConnectionError = Class.new(Gitlab::PrometheusClient::Error)
|
|
|
|
UnexpectedResponseError = Class.new(Gitlab::PrometheusClient::Error)
|
2018-02-23 12:58:40 -05:00
|
|
|
QueryError = Class.new(Gitlab::PrometheusClient::Error)
|
2020-03-25 20:07:58 -04:00
|
|
|
HEALTHY_RESPONSE = "Prometheus is Healthy.\n"
|
2018-02-23 12:58:40 -05:00
|
|
|
|
2019-04-04 11:38:37 -04:00
|
|
|
# Target number of data points for `query_range`.
|
|
|
|
# Please don't exceed the limit of 11000 data points
|
|
|
|
# See https://github.com/prometheus/prometheus/blob/91306bdf24f5395e2601773316945a478b4b263d/web/api/v1/api.go#L347
|
|
|
|
QUERY_RANGE_DATA_POINTS = 600
|
|
|
|
|
|
|
|
# Minimal value of the `step` parameter for `query_range` in seconds.
|
|
|
|
QUERY_RANGE_MIN_STEP = 60
|
|
|
|
|
2019-08-06 22:42:20 -04:00
|
|
|
# Key translation between RestClient and Gitlab::HTTP (HTTParty)
|
|
|
|
RESTCLIENT_GITLAB_HTTP_KEYMAP = {
|
|
|
|
ssl_cert_store: :cert_store
|
|
|
|
}.freeze
|
2017-03-07 11:57:42 -05:00
|
|
|
|
2019-08-06 22:42:20 -04:00
|
|
|
attr_reader :api_url, :options
|
|
|
|
private :api_url, :options
|
|
|
|
|
|
|
|
def initialize(api_url, options = {})
|
|
|
|
@api_url = api_url.chomp('/')
|
|
|
|
@options = options
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def ping
|
|
|
|
json_api_get('query', query: '1')
|
|
|
|
end
|
|
|
|
|
2020-03-25 20:07:58 -04:00
|
|
|
def healthy?
|
|
|
|
response_body = handle_management_api_response(get(health_url, {}))
|
|
|
|
|
|
|
|
# From Prometheus docs: This endpoint always returns 200 and should be used to check Prometheus health.
|
|
|
|
response_body == HEALTHY_RESPONSE
|
|
|
|
end
|
|
|
|
|
2020-09-16 14:09:47 -04:00
|
|
|
def ready?
|
|
|
|
response = get(ready_url, {})
|
|
|
|
|
|
|
|
# From Prometheus docs: This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries).
|
|
|
|
response.code == 200
|
2021-04-26 08:09:44 -04:00
|
|
|
rescue StandardError => e
|
2020-09-16 14:09:47 -04:00
|
|
|
raise PrometheusClient::UnexpectedResponseError, "#{e.message}"
|
|
|
|
end
|
|
|
|
|
2019-04-05 04:05:54 -04:00
|
|
|
def proxy(type, args)
|
|
|
|
path = api_path(type)
|
|
|
|
get(path, args)
|
2019-08-06 22:42:20 -04:00
|
|
|
rescue Gitlab::HTTP::ResponseError => ex
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::ConnectionError, "Network connection error" unless ex.response && ex.response.try(:code)
|
2019-08-06 22:42:20 -04:00
|
|
|
|
2020-03-25 20:07:58 -04:00
|
|
|
handle_querying_api_response(ex.response)
|
2019-04-05 04:05:54 -04:00
|
|
|
end
|
|
|
|
|
2017-05-09 00:15:34 -04:00
|
|
|
def query(query, time: Time.now)
|
2017-03-07 11:57:42 -05:00
|
|
|
get_result('vector') do
|
2017-04-26 16:09:03 -04:00
|
|
|
json_api_get('query', query: query, time: time.to_f)
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-03-23 20:09:24 -04:00
|
|
|
def query_range(query, start_time: 8.hours.ago, end_time: Time.now)
|
|
|
|
start_time = start_time.to_f
|
|
|
|
end_time = end_time.to_f
|
|
|
|
step = self.class.compute_step(start_time, end_time)
|
2019-04-04 11:38:37 -04:00
|
|
|
|
2017-03-07 11:57:42 -05:00
|
|
|
get_result('matrix') do
|
2019-04-04 11:38:37 -04:00
|
|
|
json_api_get(
|
|
|
|
'query_range',
|
|
|
|
query: query,
|
2020-03-23 20:09:24 -04:00
|
|
|
start: start_time,
|
|
|
|
end: end_time,
|
2019-04-04 11:38:37 -04:00
|
|
|
step: step
|
|
|
|
)
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-06-10 08:08:58 -04:00
|
|
|
# Queries Prometheus with the given aggregate query and groups the results by mapping
|
|
|
|
# metric labels to their respective values.
|
2020-06-03 08:08:21 -04:00
|
|
|
#
|
|
|
|
# @return [Hash] mapping labels to their aggregate numeric values, or the empty hash if no results were found
|
2020-08-12 08:10:25 -04:00
|
|
|
def aggregate(aggregate_query, time: Time.now, transform_value: :to_f)
|
2020-06-10 08:08:58 -04:00
|
|
|
response = query(aggregate_query, time: time)
|
2020-06-03 08:08:21 -04:00
|
|
|
response.to_h do |result|
|
2020-06-10 08:08:58 -04:00
|
|
|
key = block_given? ? yield(result['metric']) : result['metric']
|
2020-06-03 08:08:21 -04:00
|
|
|
_timestamp, value = result['value']
|
2020-08-12 08:10:25 -04:00
|
|
|
[key, value.public_send(transform_value)] # rubocop:disable GitlabSecurity/PublicSend
|
2020-06-03 08:08:21 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-06-05 15:00:57 -04:00
|
|
|
def label_values(name = '__name__')
|
2017-05-10 05:25:30 -04:00
|
|
|
json_api_get("label/#{name}/values")
|
|
|
|
end
|
|
|
|
|
2020-03-23 20:09:24 -04:00
|
|
|
def series(*matches, start_time: 8.hours.ago, end_time: Time.now)
|
|
|
|
json_api_get('series', 'match': matches, start: start_time.to_f, end: end_time.to_f)
|
2017-05-10 05:25:30 -04:00
|
|
|
end
|
|
|
|
|
2020-03-23 20:09:24 -04:00
|
|
|
def self.compute_step(start_time, end_time)
|
|
|
|
diff = end_time - start_time
|
2019-04-04 11:38:37 -04:00
|
|
|
|
|
|
|
step = (diff / QUERY_RANGE_DATA_POINTS).ceil
|
|
|
|
|
|
|
|
[QUERY_RANGE_MIN_STEP, step].max
|
|
|
|
end
|
|
|
|
|
2020-03-25 20:07:58 -04:00
|
|
|
def health_url
|
2020-09-16 14:09:47 -04:00
|
|
|
"#{api_url}/-/healthy"
|
|
|
|
end
|
|
|
|
|
|
|
|
def ready_url
|
|
|
|
"#{api_url}/-/ready"
|
2020-03-25 20:07:58 -04:00
|
|
|
end
|
|
|
|
|
2017-03-07 11:57:42 -05:00
|
|
|
private
|
|
|
|
|
2019-04-05 04:05:54 -04:00
|
|
|
def api_path(type)
|
2019-08-06 22:42:20 -04:00
|
|
|
[api_url, 'api', 'v1', type].join('/')
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
|
2019-04-05 04:05:54 -04:00
|
|
|
def json_api_get(type, args = {})
|
|
|
|
path = api_path(type)
|
|
|
|
response = get(path, args)
|
2020-03-25 20:07:58 -04:00
|
|
|
handle_querying_api_response(response)
|
2019-08-06 22:42:20 -04:00
|
|
|
rescue Gitlab::HTTP::ResponseError => ex
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::ConnectionError, "Network connection error" unless ex.response && ex.response.try(:code)
|
2019-08-06 22:42:20 -04:00
|
|
|
|
2020-03-25 20:07:58 -04:00
|
|
|
handle_querying_api_response(ex.response)
|
2019-08-06 22:42:20 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def gitlab_http_key(key)
|
|
|
|
RESTCLIENT_GITLAB_HTTP_KEYMAP[key] || key
|
|
|
|
end
|
|
|
|
|
|
|
|
def mapped_options
|
2021-04-05 14:09:15 -04:00
|
|
|
options.keys.to_h { |k| [gitlab_http_key(k), options[k]] }
|
2019-08-06 22:42:20 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def http_options
|
|
|
|
strong_memoize(:http_options) do
|
2020-08-13 17:10:04 -04:00
|
|
|
{ follow_redirects: false }.merge(mapped_options)
|
2018-02-23 15:33:33 -05:00
|
|
|
end
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
|
2019-04-05 04:05:54 -04:00
|
|
|
def get(path, args)
|
2019-08-06 22:42:20 -04:00
|
|
|
Gitlab::HTTP.get(path, { query: args }.merge(http_options) )
|
2019-04-05 04:05:54 -04:00
|
|
|
rescue SocketError
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::ConnectionError, "Can't connect to #{api_url}"
|
2019-04-05 04:05:54 -04:00
|
|
|
rescue OpenSSL::SSL::SSLError
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::ConnectionError, "#{api_url} contains invalid SSL data"
|
2019-04-05 04:05:54 -04:00
|
|
|
rescue Errno::ECONNREFUSED
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::ConnectionError, 'Connection refused'
|
2019-04-05 04:05:54 -04:00
|
|
|
end
|
|
|
|
|
2020-03-25 20:07:58 -04:00
|
|
|
def handle_management_api_response(response)
|
|
|
|
if response.code == 200
|
|
|
|
response.body
|
|
|
|
else
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::UnexpectedResponseError, "#{response.code} - #{response.body}"
|
2020-03-25 20:07:58 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def handle_querying_api_response(response)
|
2019-08-06 22:42:20 -04:00
|
|
|
response_code = response.try(:code)
|
|
|
|
response_body = response.try(:body)
|
|
|
|
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::UnexpectedResponseError, "#{response_code} - #{response_body}" unless response_code
|
2019-08-06 22:42:20 -04:00
|
|
|
|
|
|
|
json_data = parse_json(response_body) if [200, 400].include?(response_code)
|
2018-01-03 21:13:54 -05:00
|
|
|
|
2019-08-06 22:42:20 -04:00
|
|
|
case response_code
|
|
|
|
when 200
|
|
|
|
json_data['data'] if response['status'] == 'success'
|
|
|
|
when 400
|
2018-02-23 12:58:40 -05:00
|
|
|
raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received'
|
2017-03-07 11:57:42 -05:00
|
|
|
else
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::UnexpectedResponseError, "#{response_code} - #{response_body}"
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def get_result(expected_type)
|
|
|
|
data = yield
|
|
|
|
data['result'] if data['resultType'] == expected_type
|
|
|
|
end
|
2019-04-05 04:05:54 -04:00
|
|
|
|
|
|
|
def parse_json(response_body)
|
2020-05-08 05:09:39 -04:00
|
|
|
Gitlab::Json.parse(response_body, legacy_mode: true)
|
2019-04-05 04:05:54 -04:00
|
|
|
rescue JSON::ParserError
|
2020-07-03 11:09:13 -04:00
|
|
|
raise PrometheusClient::UnexpectedResponseError, 'Parsing response failed'
|
2019-04-05 04:05:54 -04:00
|
|
|
end
|
2017-03-07 11:57:42 -05:00
|
|
|
end
|
|
|
|
end
|