Merge branch 'pawel/update_prometheus_gem_to_well_tested_version' into 'master'
Update Prometheus Gem version and disable Prometheus method call instrumentation by default. Closes gitlab-ee#4139 and #40457 See merge request gitlab-org/gitlab-ce!15558
This commit is contained in:
commit
fad4ab7d56
2
Gemfile
2
Gemfile
|
@ -283,7 +283,7 @@ group :metrics do
|
||||||
gem 'influxdb', '~> 0.2', require: false
|
gem 'influxdb', '~> 0.2', require: false
|
||||||
|
|
||||||
# Prometheus
|
# Prometheus
|
||||||
gem 'prometheus-client-mmap', '~>0.7.0.beta18'
|
gem 'prometheus-client-mmap', '~> 0.7.0.beta36'
|
||||||
gem 'raindrops', '~> 0.18'
|
gem 'raindrops', '~> 0.18'
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -488,7 +488,7 @@ GEM
|
||||||
mini_mime (0.1.4)
|
mini_mime (0.1.4)
|
||||||
mini_portile2 (2.3.0)
|
mini_portile2 (2.3.0)
|
||||||
minitest (5.7.0)
|
minitest (5.7.0)
|
||||||
mmap2 (2.2.7)
|
mmap2 (2.2.9)
|
||||||
mousetrap-rails (1.4.6)
|
mousetrap-rails (1.4.6)
|
||||||
multi_json (1.12.2)
|
multi_json (1.12.2)
|
||||||
multi_xml (0.6.0)
|
multi_xml (0.6.0)
|
||||||
|
@ -625,8 +625,8 @@ GEM
|
||||||
parser
|
parser
|
||||||
unparser
|
unparser
|
||||||
procto (0.0.3)
|
procto (0.0.3)
|
||||||
prometheus-client-mmap (0.7.0.beta18)
|
prometheus-client-mmap (0.7.0.beta36)
|
||||||
mmap2 (~> 2.2, >= 2.2.7)
|
mmap2 (~> 2.2, >= 2.2.9)
|
||||||
pry (0.10.4)
|
pry (0.10.4)
|
||||||
coderay (~> 1.1.0)
|
coderay (~> 1.1.0)
|
||||||
method_source (~> 0.8.1)
|
method_source (~> 0.8.1)
|
||||||
|
@ -1111,7 +1111,7 @@ DEPENDENCIES
|
||||||
peek-sidekiq (~> 1.0.3)
|
peek-sidekiq (~> 1.0.3)
|
||||||
pg (~> 0.18.2)
|
pg (~> 0.18.2)
|
||||||
premailer-rails (~> 1.9.7)
|
premailer-rails (~> 1.9.7)
|
||||||
prometheus-client-mmap (~> 0.7.0.beta18)
|
prometheus-client-mmap (~> 0.7.0.beta36)
|
||||||
pry-byebug (~> 3.4.1)
|
pry-byebug (~> 3.4.1)
|
||||||
pry-rails (~> 0.3.4)
|
pry-rails (~> 0.3.4)
|
||||||
rack-attack (~> 4.4.1)
|
rack-attack (~> 4.4.1)
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
title: Reenable Prometheus metrics, add more control over Prometheus method instrumentation
|
||||||
|
merge_request: 15558
|
||||||
|
author:
|
||||||
|
type: fixed
|
|
@ -12,16 +12,20 @@ Prometheus::Client.configure do |config|
|
||||||
end
|
end
|
||||||
|
|
||||||
config.pid_provider = -> do
|
config.pid_provider = -> do
|
||||||
wid = Prometheus::Client::Support::Unicorn.worker_id
|
worker_id = Prometheus::Client::Support::Unicorn.worker_id
|
||||||
wid = Process.pid if wid.nil?
|
if worker_id.nil?
|
||||||
if wid.nil?
|
|
||||||
"process_pid_#{Process.pid}"
|
"process_pid_#{Process.pid}"
|
||||||
else
|
else
|
||||||
"worker_id_#{wid}"
|
"worker_id_#{worker_id}"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
Gitlab::Application.configure do |config|
|
||||||
|
# 0 should be Sentry to catch errors in this middleware
|
||||||
|
config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
|
||||||
|
end
|
||||||
|
|
||||||
Sidekiq.configure_server do |config|
|
Sidekiq.configure_server do |config|
|
||||||
config.on(:startup) do
|
config.on(:startup) do
|
||||||
Gitlab::Metrics::SidekiqMetricsExporter.instance.start
|
Gitlab::Metrics::SidekiqMetricsExporter.instance.start
|
||||||
|
|
|
@ -118,11 +118,6 @@ def instrument_classes(instrumentation)
|
||||||
end
|
end
|
||||||
# rubocop:enable Metrics/AbcSize
|
# rubocop:enable Metrics/AbcSize
|
||||||
|
|
||||||
Gitlab::Application.configure do |config|
|
|
||||||
# 0 should be Sentry to catch errors in this middleware
|
|
||||||
config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
|
|
||||||
end
|
|
||||||
|
|
||||||
if Gitlab::Metrics.enabled?
|
if Gitlab::Metrics.enabled?
|
||||||
require 'pathname'
|
require 'pathname'
|
||||||
require 'influxdb'
|
require 'influxdb'
|
||||||
|
|
|
@ -6,29 +6,15 @@ module Gitlab
|
||||||
BASE_LABELS = { module: nil, method: nil }.freeze
|
BASE_LABELS = { module: nil, method: nil }.freeze
|
||||||
attr_reader :real_time, :cpu_time, :call_count, :labels
|
attr_reader :real_time, :cpu_time, :call_count, :labels
|
||||||
|
|
||||||
def self.call_real_duration_histogram
|
def self.call_duration_histogram
|
||||||
return @call_real_duration_histogram if @call_real_duration_histogram
|
return @call_duration_histogram if @call_duration_histogram
|
||||||
|
|
||||||
MUTEX.synchronize do
|
|
||||||
@call_real_duration_histogram ||= Gitlab::Metrics.histogram(
|
|
||||||
:gitlab_method_call_real_duration_seconds,
|
|
||||||
'Method calls real duration',
|
|
||||||
Transaction::BASE_LABELS.merge(BASE_LABELS),
|
|
||||||
[0.1, 0.2, 0.5, 1, 2, 5, 10]
|
|
||||||
)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.call_cpu_duration_histogram
|
|
||||||
return @call_cpu_duration_histogram if @call_cpu_duration_histogram
|
|
||||||
|
|
||||||
MUTEX.synchronize do
|
MUTEX.synchronize do
|
||||||
@call_duration_histogram ||= Gitlab::Metrics.histogram(
|
@call_duration_histogram ||= Gitlab::Metrics.histogram(
|
||||||
:gitlab_method_call_cpu_duration_seconds,
|
:gitlab_method_call_duration_seconds,
|
||||||
'Method calls cpu duration',
|
'Method calls real duration',
|
||||||
Transaction::BASE_LABELS.merge(BASE_LABELS),
|
Transaction::BASE_LABELS.merge(BASE_LABELS),
|
||||||
[0.1, 0.2, 0.5, 1, 2, 5, 10]
|
[0.01, 0.05, 0.1, 0.5, 1])
|
||||||
)
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -59,8 +45,9 @@ module Gitlab
|
||||||
@cpu_time += cpu_time
|
@cpu_time += cpu_time
|
||||||
@call_count += 1
|
@call_count += 1
|
||||||
|
|
||||||
self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0)
|
if call_measurement_enabled? && above_threshold?
|
||||||
self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0)
|
self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0)
|
||||||
|
end
|
||||||
|
|
||||||
retval
|
retval
|
||||||
end
|
end
|
||||||
|
@ -83,6 +70,10 @@ module Gitlab
|
||||||
def above_threshold?
|
def above_threshold?
|
||||||
real_time >= Metrics.method_call_threshold
|
real_time >= Metrics.method_call_threshold
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def call_measurement_enabled?
|
||||||
|
Feature.get(:prometheus_metrics_method_instrumentation).enabled?
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -17,9 +17,9 @@ module Gitlab
|
||||||
end
|
end
|
||||||
|
|
||||||
def prometheus_metrics_enabled?
|
def prometheus_metrics_enabled?
|
||||||
# force disable prometheus_metrics until
|
return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled)
|
||||||
# https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/11 is ready
|
|
||||||
false
|
@prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized
|
||||||
end
|
end
|
||||||
|
|
||||||
def registry
|
def registry
|
||||||
|
|
|
@ -13,16 +13,52 @@ describe Gitlab::Metrics::MethodCall do
|
||||||
expect(method_call.call_count).to eq(1)
|
expect(method_call.call_count).to eq(1)
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'observes the performance of the supplied block' do
|
context 'when measurement is above threshold' do
|
||||||
expect(described_class.call_real_duration_histogram)
|
before do
|
||||||
.to receive(:observe)
|
allow(method_call).to receive(:above_threshold?).and_return(true)
|
||||||
.with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric))
|
end
|
||||||
|
|
||||||
expect(described_class.call_cpu_duration_histogram)
|
context 'prometheus instrumentation is enabled' do
|
||||||
.to receive(:observe)
|
before do
|
||||||
.with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric))
|
Feature.get(:prometheus_metrics_method_instrumentation).enable
|
||||||
|
end
|
||||||
|
|
||||||
method_call.measure { 'foo' }
|
it 'observes the performance of the supplied block' do
|
||||||
|
expect(described_class.call_duration_histogram)
|
||||||
|
.to receive(:observe)
|
||||||
|
.with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric))
|
||||||
|
|
||||||
|
method_call.measure { 'foo' }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'prometheus instrumentation is disabled' do
|
||||||
|
before do
|
||||||
|
Feature.get(:prometheus_metrics_method_instrumentation).disable
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not observe the performance' do
|
||||||
|
expect(described_class.call_duration_histogram)
|
||||||
|
.not_to receive(:observe)
|
||||||
|
|
||||||
|
method_call.measure { 'foo' }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when measurement is below threshold' do
|
||||||
|
before do
|
||||||
|
allow(method_call).to receive(:above_threshold?).and_return(false)
|
||||||
|
|
||||||
|
Feature.get(:prometheus_metrics_method_instrumentation).enable
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not observe the performance' do
|
||||||
|
expect(described_class.call_duration_histogram)
|
||||||
|
.not_to receive(:observe)
|
||||||
|
|
||||||
|
method_call.measure { 'foo' }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -43,7 +79,13 @@ describe Gitlab::Metrics::MethodCall do
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '#above_threshold?' do
|
describe '#above_threshold?' do
|
||||||
|
before do
|
||||||
|
allow(Gitlab::Metrics).to receive(:method_call_threshold).and_return(100)
|
||||||
|
end
|
||||||
|
|
||||||
it 'returns false when the total call time is not above the threshold' do
|
it 'returns false when the total call time is not above the threshold' do
|
||||||
|
expect(method_call).to receive(:real_time).and_return(9)
|
||||||
|
|
||||||
expect(method_call.above_threshold?).to eq(false)
|
expect(method_call.above_threshold?).to eq(false)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue