Split the metrics implementation to separate modules for Influx and Prometheus
This commit is contained in:
parent
ae8f7666e5
commit
b668aaf426
|
@ -30,14 +30,4 @@ class MetricsService
|
||||||
def multiprocess_metrics_path
|
def multiprocess_metrics_path
|
||||||
@multiprocess_metrics_path ||= Rails.root.join(ENV['prometheus_multiproc_dir']).freeze
|
@multiprocess_metrics_path ||= Rails.root.join(ENV['prometheus_multiproc_dir']).freeze
|
||||||
end
|
end
|
||||||
|
|
||||||
def metric_to_prom_line(metric)
|
|
||||||
labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
|
|
||||||
|
|
||||||
if labels.empty?
|
|
||||||
"#{metric.name} #{metric.value}"
|
|
||||||
else
|
|
||||||
"#{metric.name}{#{labels}} #{metric.value}"
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,38 +1,40 @@
|
||||||
module Gitlab::HealthChecks
|
module Gitlab
|
||||||
class PrometheusTextFormat
|
module HealthChecks
|
||||||
def marshal(metrics)
|
class PrometheusTextFormat
|
||||||
metrics_with_type_declarations(metrics).join("\n")
|
def marshal(metrics)
|
||||||
end
|
metrics_with_type_declarations(metrics).join("\n")
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def metrics_with_type_declarations(metrics)
|
|
||||||
type_declaration_added = {}
|
|
||||||
|
|
||||||
metrics.flat_map do |metric|
|
|
||||||
metric_lines = []
|
|
||||||
|
|
||||||
unless type_declaration_added.has_key?(metric.name)
|
|
||||||
type_declaration_added[metric.name] = true
|
|
||||||
metric_lines << metric_type_declaration(metric)
|
|
||||||
end
|
|
||||||
|
|
||||||
metric_lines << metric_text(metric)
|
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
def metric_type_declaration(metric)
|
private
|
||||||
"# TYPE #{metric.name} gauge"
|
|
||||||
end
|
|
||||||
|
|
||||||
def metric_text(metric)
|
def metrics_with_type_declarations(metrics)
|
||||||
labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
|
type_declaration_added = {}
|
||||||
|
|
||||||
if labels.empty?
|
metrics.flat_map do |metric|
|
||||||
"#{metric.name} #{metric.value}"
|
metric_lines = []
|
||||||
else
|
|
||||||
"#{metric.name}{#{labels}} #{metric.value}"
|
unless type_declaration_added.has_key?(metric.name)
|
||||||
|
type_declaration_added[metric.name] = true
|
||||||
|
metric_lines << metric_type_declaration(metric)
|
||||||
|
end
|
||||||
|
|
||||||
|
metric_lines << metric_text(metric)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def metric_type_declaration(metric)
|
||||||
|
"# TYPE #{metric.name} gauge"
|
||||||
|
end
|
||||||
|
|
||||||
|
def metric_text(metric)
|
||||||
|
labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
|
||||||
|
|
||||||
|
if labels.empty?
|
||||||
|
"#{metric.name} #{metric.value}"
|
||||||
|
else
|
||||||
|
"#{metric.name}{#{labels}} #{metric.value}"
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -1,200 +1,10 @@
|
||||||
require 'prometheus/client'
|
|
||||||
|
|
||||||
module Gitlab
|
module Gitlab
|
||||||
module Metrics
|
module Metrics
|
||||||
extend Gitlab::CurrentSettings
|
extend Gitlab::Metrics::InfluxDb
|
||||||
|
extend Gitlab::Metrics::Prometheus
|
||||||
RAILS_ROOT = Rails.root.to_s
|
|
||||||
METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
|
|
||||||
PATH_REGEX = /^#{RAILS_ROOT}\/?/
|
|
||||||
|
|
||||||
def self.settings
|
|
||||||
@settings ||= {
|
|
||||||
enabled: current_application_settings[:metrics_enabled],
|
|
||||||
prometheus_metrics_enabled: current_application_settings[:prometheus_metrics_enabled],
|
|
||||||
pool_size: current_application_settings[:metrics_pool_size],
|
|
||||||
timeout: current_application_settings[:metrics_timeout],
|
|
||||||
method_call_threshold: current_application_settings[:metrics_method_call_threshold],
|
|
||||||
host: current_application_settings[:metrics_host],
|
|
||||||
port: current_application_settings[:metrics_port],
|
|
||||||
sample_interval: current_application_settings[:metrics_sample_interval] || 15,
|
|
||||||
packet_size: current_application_settings[:metrics_packet_size] || 1
|
|
||||||
}
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.prometheus_metrics_enabled?
|
|
||||||
settings[:prometheus_metrics_enabled] || false
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.influx_metrics_enabled?
|
|
||||||
settings[:enabled] || false
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.enabled?
|
def self.enabled?
|
||||||
influx_metrics_enabled? || prometheus_metrics_enabled?
|
influx_metrics_enabled? || prometheus_metrics_enabled?
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.mri?
|
|
||||||
RUBY_ENGINE == 'ruby'
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.method_call_threshold
|
|
||||||
# This is memoized since this method is called for every instrumented
|
|
||||||
# method. Loading data from an external cache on every method call slows
|
|
||||||
# things down too much.
|
|
||||||
@method_call_threshold ||= settings[:method_call_threshold]
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.pool
|
|
||||||
@pool
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.registry
|
|
||||||
@registry ||= ::Prometheus::Client.registry
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.counter(name, docstring, base_labels = {})
|
|
||||||
provide_metric(name) || registry.counter(name, docstring, base_labels)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.summary(name, docstring, base_labels = {})
|
|
||||||
provide_metric(name) || registry.summary(name, docstring, base_labels)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.gauge(name, docstring, base_labels = {})
|
|
||||||
provide_metric(name) || registry.gauge(name, docstring, base_labels)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
|
|
||||||
provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.provide_metric(name)
|
|
||||||
if prometheus_metrics_enabled?
|
|
||||||
registry.get(name)
|
|
||||||
else
|
|
||||||
NullMetric.new
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.submit_metrics(metrics)
|
|
||||||
prepared = prepare_metrics(metrics)
|
|
||||||
|
|
||||||
pool&.with do |connection|
|
|
||||||
prepared.each_slice(settings[:packet_size]) do |slice|
|
|
||||||
begin
|
|
||||||
connection.write_points(slice)
|
|
||||||
rescue StandardError
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
rescue Errno::EADDRNOTAVAIL, SocketError => ex
|
|
||||||
Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
|
|
||||||
Gitlab::EnvironmentLogger.error(ex)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.prepare_metrics(metrics)
|
|
||||||
metrics.map do |hash|
|
|
||||||
new_hash = hash.symbolize_keys
|
|
||||||
|
|
||||||
new_hash[:tags].each do |key, value|
|
|
||||||
if value.blank?
|
|
||||||
new_hash[:tags].delete(key)
|
|
||||||
else
|
|
||||||
new_hash[:tags][key] = escape_value(value)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
new_hash
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.escape_value(value)
|
|
||||||
value.to_s.gsub('=', '\\=')
|
|
||||||
end
|
|
||||||
|
|
||||||
# Measures the execution time of a block.
|
|
||||||
#
|
|
||||||
# Example:
|
|
||||||
#
|
|
||||||
# Gitlab::Metrics.measure(:find_by_username_duration) do
|
|
||||||
# User.find_by_username(some_username)
|
|
||||||
# end
|
|
||||||
#
|
|
||||||
# name - The name of the field to store the execution time in.
|
|
||||||
#
|
|
||||||
# Returns the value yielded by the supplied block.
|
|
||||||
def self.measure(name)
|
|
||||||
trans = current_transaction
|
|
||||||
|
|
||||||
return yield unless trans
|
|
||||||
|
|
||||||
real_start = Time.now.to_f
|
|
||||||
cpu_start = System.cpu_time
|
|
||||||
|
|
||||||
retval = yield
|
|
||||||
|
|
||||||
cpu_stop = System.cpu_time
|
|
||||||
real_stop = Time.now.to_f
|
|
||||||
|
|
||||||
real_time = (real_stop - real_start) * 1000.0
|
|
||||||
cpu_time = cpu_stop - cpu_start
|
|
||||||
|
|
||||||
trans.increment("#{name}_real_time", real_time)
|
|
||||||
trans.increment("#{name}_cpu_time", cpu_time)
|
|
||||||
trans.increment("#{name}_call_count", 1)
|
|
||||||
|
|
||||||
retval
|
|
||||||
end
|
|
||||||
|
|
||||||
# Adds a tag to the current transaction (if any)
|
|
||||||
#
|
|
||||||
# name - The name of the tag to add.
|
|
||||||
# value - The value of the tag.
|
|
||||||
def self.tag_transaction(name, value)
|
|
||||||
trans = current_transaction
|
|
||||||
|
|
||||||
trans&.add_tag(name, value)
|
|
||||||
end
|
|
||||||
|
|
||||||
# Sets the action of the current transaction (if any)
|
|
||||||
#
|
|
||||||
# action - The name of the action.
|
|
||||||
def self.action=(action)
|
|
||||||
trans = current_transaction
|
|
||||||
|
|
||||||
trans&.action = action
|
|
||||||
end
|
|
||||||
|
|
||||||
# Tracks an event.
|
|
||||||
#
|
|
||||||
# See `Gitlab::Metrics::Transaction#add_event` for more details.
|
|
||||||
def self.add_event(*args)
|
|
||||||
trans = current_transaction
|
|
||||||
|
|
||||||
trans&.add_event(*args)
|
|
||||||
end
|
|
||||||
|
|
||||||
# Returns the prefix to use for the name of a series.
|
|
||||||
def self.series_prefix
|
|
||||||
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
|
|
||||||
end
|
|
||||||
|
|
||||||
# Allow access from other metrics related middlewares
|
|
||||||
def self.current_transaction
|
|
||||||
Transaction.current
|
|
||||||
end
|
|
||||||
|
|
||||||
# When enabled this should be set before being used as the usual pattern
|
|
||||||
# "@foo ||= bar" is _not_ thread-safe.
|
|
||||||
if influx_metrics_enabled?
|
|
||||||
@pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
|
|
||||||
host = settings[:host]
|
|
||||||
port = settings[:port]
|
|
||||||
|
|
||||||
InfluxDB::Client.
|
|
||||||
new(udp: { host: host, port: port })
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -0,0 +1,163 @@
|
||||||
|
module Gitlab
|
||||||
|
module Metrics
|
||||||
|
module InfluxDb
|
||||||
|
include Gitlab::CurrentSettings
|
||||||
|
|
||||||
|
def influx_metrics_enabled?
|
||||||
|
settings[:enabled] || false
|
||||||
|
end
|
||||||
|
|
||||||
|
RAILS_ROOT = Rails.root.to_s
|
||||||
|
METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
|
||||||
|
PATH_REGEX = /^#{RAILS_ROOT}\/?/
|
||||||
|
|
||||||
|
def settings
|
||||||
|
@settings ||= {
|
||||||
|
enabled: current_application_settings[:metrics_enabled],
|
||||||
|
pool_size: current_application_settings[:metrics_pool_size],
|
||||||
|
timeout: current_application_settings[:metrics_timeout],
|
||||||
|
method_call_threshold: current_application_settings[:metrics_method_call_threshold],
|
||||||
|
host: current_application_settings[:metrics_host],
|
||||||
|
port: current_application_settings[:metrics_port],
|
||||||
|
sample_interval: current_application_settings[:metrics_sample_interval] || 15,
|
||||||
|
packet_size: current_application_settings[:metrics_packet_size] || 1
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def mri?
|
||||||
|
RUBY_ENGINE == 'ruby'
|
||||||
|
end
|
||||||
|
|
||||||
|
def method_call_threshold
|
||||||
|
# This is memoized since this method is called for every instrumented
|
||||||
|
# method. Loading data from an external cache on every method call slows
|
||||||
|
# things down too much.
|
||||||
|
@method_call_threshold ||= settings[:method_call_threshold]
|
||||||
|
end
|
||||||
|
|
||||||
|
def pool
|
||||||
|
@pool
|
||||||
|
end
|
||||||
|
|
||||||
|
def submit_metrics(metrics)
|
||||||
|
prepared = prepare_metrics(metrics)
|
||||||
|
|
||||||
|
pool&.with do |connection|
|
||||||
|
prepared.each_slice(settings[:packet_size]) do |slice|
|
||||||
|
begin
|
||||||
|
connection.write_points(slice)
|
||||||
|
rescue StandardError
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
rescue Errno::EADDRNOTAVAIL, SocketError => ex
|
||||||
|
Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
|
||||||
|
Gitlab::EnvironmentLogger.error(ex)
|
||||||
|
end
|
||||||
|
|
||||||
|
def prepare_metrics(metrics)
|
||||||
|
metrics.map do |hash|
|
||||||
|
new_hash = hash.symbolize_keys
|
||||||
|
|
||||||
|
new_hash[:tags].each do |key, value|
|
||||||
|
if value.blank?
|
||||||
|
new_hash[:tags].delete(key)
|
||||||
|
else
|
||||||
|
new_hash[:tags][key] = escape_value(value)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
new_hash
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def escape_value(value)
|
||||||
|
value.to_s.gsub('=', '\\=')
|
||||||
|
end
|
||||||
|
|
||||||
|
# Measures the execution time of a block.
|
||||||
|
#
|
||||||
|
# Example:
|
||||||
|
#
|
||||||
|
# Gitlab::Metrics.measure(:find_by_username_duration) do
|
||||||
|
# User.find_by_username(some_username)
|
||||||
|
# end
|
||||||
|
#
|
||||||
|
# name - The name of the field to store the execution time in.
|
||||||
|
#
|
||||||
|
# Returns the value yielded by the supplied block.
|
||||||
|
def measure(name)
|
||||||
|
trans = current_transaction
|
||||||
|
|
||||||
|
return yield unless trans
|
||||||
|
|
||||||
|
real_start = Time.now.to_f
|
||||||
|
cpu_start = System.cpu_time
|
||||||
|
|
||||||
|
retval = yield
|
||||||
|
|
||||||
|
cpu_stop = System.cpu_time
|
||||||
|
real_stop = Time.now.to_f
|
||||||
|
|
||||||
|
real_time = (real_stop - real_start) * 1000.0
|
||||||
|
cpu_time = cpu_stop - cpu_start
|
||||||
|
|
||||||
|
trans.increment("#{name}_real_time", real_time)
|
||||||
|
trans.increment("#{name}_cpu_time", cpu_time)
|
||||||
|
trans.increment("#{name}_call_count", 1)
|
||||||
|
|
||||||
|
retval
|
||||||
|
end
|
||||||
|
|
||||||
|
# Adds a tag to the current transaction (if any)
|
||||||
|
#
|
||||||
|
# name - The name of the tag to add.
|
||||||
|
# value - The value of the tag.
|
||||||
|
def tag_transaction(name, value)
|
||||||
|
trans = current_transaction
|
||||||
|
|
||||||
|
trans&.add_tag(name, value)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Sets the action of the current transaction (if any)
|
||||||
|
#
|
||||||
|
# action - The name of the action.
|
||||||
|
def action=(action)
|
||||||
|
trans = current_transaction
|
||||||
|
|
||||||
|
trans&.action = action
|
||||||
|
end
|
||||||
|
|
||||||
|
# Tracks an event.
|
||||||
|
#
|
||||||
|
# See `Gitlab::Metrics::Transaction#add_event` for more details.
|
||||||
|
def add_event(*args)
|
||||||
|
trans = current_transaction
|
||||||
|
|
||||||
|
trans&.add_event(*args)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Returns the prefix to use for the name of a series.
|
||||||
|
def series_prefix
|
||||||
|
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
|
||||||
|
end
|
||||||
|
|
||||||
|
# Allow access from other metrics related middlewares
|
||||||
|
def current_transaction
|
||||||
|
Transaction.current
|
||||||
|
end
|
||||||
|
|
||||||
|
# When enabled this should be set before being used as the usual pattern
|
||||||
|
# "@foo ||= bar" is _not_ thread-safe.
|
||||||
|
if influx_metrics_enabled?
|
||||||
|
@pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
|
||||||
|
host = settings[:host]
|
||||||
|
port = settings[:port]
|
||||||
|
|
||||||
|
InfluxDB::Client.
|
||||||
|
new(udp: { host: host, port: port })
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,41 @@
|
||||||
|
require 'prometheus/client'
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module Metrics
|
||||||
|
module Prometheus
|
||||||
|
include Gitlab::CurrentSettings
|
||||||
|
|
||||||
|
def prometheus_metrics_enabled?
|
||||||
|
@prometheus_metrics_enabled ||= current_application_settings[:prometheus_metrics_enabled] || false
|
||||||
|
end
|
||||||
|
|
||||||
|
def registry
|
||||||
|
@registry ||= ::Prometheus::Client.registry
|
||||||
|
end
|
||||||
|
|
||||||
|
def counter(name, docstring, base_labels = {})
|
||||||
|
provide_metric(name) || registry.counter(name, docstring, base_labels)
|
||||||
|
end
|
||||||
|
|
||||||
|
def summary(name, docstring, base_labels = {})
|
||||||
|
provide_metric(name) || registry.summary(name, docstring, base_labels)
|
||||||
|
end
|
||||||
|
|
||||||
|
def gauge(name, docstring, base_labels = {})
|
||||||
|
provide_metric(name) || registry.gauge(name, docstring, base_labels)
|
||||||
|
end
|
||||||
|
|
||||||
|
def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
|
||||||
|
provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets)
|
||||||
|
end
|
||||||
|
|
||||||
|
def provide_metric(name)
|
||||||
|
if prometheus_metrics_enabled?
|
||||||
|
registry.get(name)
|
||||||
|
else
|
||||||
|
NullMetric.new
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue