kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Update docs and calculate process start time via proc table

Browse source 
This updates monitor docs to reflect the new ruby and unicorn metrics as
well as making it so we fetch process start time via the proc table
instead of via CLOCK_BOOTTIME
This commit is contained in:
Ryan Cobb 2019-04-24 17:05:09 -06:00
parent 174a03dfc2
commit bb27bf4a1c
5 changed files with 44 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
doc/administration/monitoring/prometheus/gitlab_metrics.md
View file

@ -43,10 +43,11 @@ The following metrics are available:
| redis_ping_latency_seconds | Gauge | 9.4 | Round trip time of the redis ping |
| user_session_logins_total | Counter | 9.4 | Counter of how many users have logged in |
| upload_file_does_not_exist | Counter | 10.7 in EE, 11.5 in CE | Number of times an upload record could not find its file |
| failed_login_captcha_total | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login |
| successful_login_captcha_total | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login |
| unicorn_active_connections | Gauge | 11.0 | The number of active Unicorn connections (workers) |
| unicorn_queued_connections | Gauge | 11.0 | The number of queued Unicorn connections |
| failed_login_captcha_total | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login |
| successful_login_captcha_total | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login |
| unicorn_active_connections | Gauge | 11.0 | The number of active Unicorn connections (workers) |
| unicorn_queued_connections | Gauge | 11.0 | The number of queued Unicorn connections |
| unicorn_workers | Gauge | 11.11 | The number of Unicorn workers |
### Ruby metrics
@ -57,8 +58,11 @@ Some basic Ruby runtime metrics are available:
| ruby_gc_duration_seconds_total | Counter | 11.1 | Time spent by Ruby in GC |
| ruby_gc_stat_... | Gauge | 11.1 | Various metrics from [GC.stat] |
| ruby_file_descriptors | Gauge | 11.1 | File descriptors per process |
| ruby_memory_bytes | Gauge | 11.1 | Memory usage by process |
| ruby_process_resident_memory_bytes | Gauge | 11.1 | Memory usage by process |
| ruby_sampler_duration_seconds_total | Counter | 11.1 | Time spent collecting stats |
| ruby_process_cpu_seconds_total | Gauge | 11.11 | Total amount of cpu time per process |
| ruby_process_max_fds | Gauge | 11.11 | Maximum number of open file descriptors per process |
| ruby_process_start_time_seconds | Gauge | 11.11 | The time the process started after system boot in seconds |
[GC.stat]: https://ruby-doc.org/core-2.3.0/GC.html#method-c-stat

18
lib/gitlab/metrics/samplers/ruby_sampler.rb
View file

@ -24,13 +24,13 @@ module Gitlab
def init_metrics
metrics = {
file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
memory_usage: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum),
process_cpu_seconds_total: ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'),
process_max_fds: ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'),
process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds'),
sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum),
process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
}
GC.stat.keys.each do |key|
@ -44,10 +44,10 @@ module Gitlab
start_time = System.monotonic_time
metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time)
metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors)
metrics[:process_resident_memory_bytes].set(labels.merge(worker_label), System.memory_usage)
metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
sample_gc
metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)

22
lib/gitlab/metrics/system.rb
View file

@ -33,6 +33,13 @@ module Gitlab
max_fds
end
def self.process_start_time
start_time_in_jiffies = Sys::ProcTable.ps(pid: Process.pid).starttime
return 0 unless start_time_in_jiffies
start_time_in_jiffies / 100
end
else
def self.memory_usage
0.0
@ -45,6 +52,10 @@ module Gitlab
def self.max_open_file_descriptors
0
end
def self.process_start_time
0
end
end
# THREAD_CPUTIME is not supported on OS X
@ -60,17 +71,6 @@ module Gitlab
end
end
# CLOCK_BOOTTIME is not supported on OS X
if Process.const_defined?(:CLOCK_BOOTTIME)
def self.process_start_time
Process
.clock_gettime(Process::CLOCK_BOOTTIME, :float_second)
end
else
def self.process_start_time
0.0
end
end
# Returns the current real time in a given precision.
#
# Returns the time as a Float for precision = :float_second.

6
spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb
View file

@ -20,10 +20,10 @@ describe Gitlab::Metrics::Samplers::RubySampler do
sampler.sample
end
it 'adds a metric containing the memory usage' do
it 'adds a metric containing the process resident memory bytes' do
expect(Gitlab::Metrics::System).to receive(:memory_usage).and_return(9000)
expect(sampler.metrics[:memory_usage]).to receive(:set).with({}, 9000)
expect(sampler.metrics[:process_resident_memory_bytes]).to receive(:set).with({}, 9000)
sampler.sample
end
@ -37,7 +37,7 @@ describe Gitlab::Metrics::Samplers::RubySampler do
sampler.sample
end
it 'adds a metric containing the processes total cpu time' do
it 'adds a metric containing the process total cpu time' do
expect(Gitlab::Metrics::System).to receive(:cpu_time).and_return(0.51)
expect(sampler.metrics[:process_cpu_seconds_total]).to receive(:set).with({}, 0.51)

18
spec/lib/gitlab/metrics/system_spec.rb
View file

@ -19,6 +19,12 @@ describe Gitlab::Metrics::System do
expect(described_class.max_open_file_descriptors).to be > 0
end
end
describe '.process_start_time' do
it 'returns the process start time' do
expect(described_class.process_start_time).to be > 0
end
end
else
describe '.memory_usage' do
it 'returns 0.0' do
@ -37,6 +43,12 @@ describe Gitlab::Metrics::System do
expect(described_class.max_open_file_descriptors).to eq(0)
end
end
describe 'process_start_time' do
it 'returns 0' do
expect(described_class.process_start_time).to eq(0)
end
end
end
describe '.cpu_time' do
@ -56,10 +68,4 @@ describe Gitlab::Metrics::System do
expect(described_class.monotonic_time).to be_an(Float)
end
end
describe '.process_start_time' do
it 'returns a Float' do
expect(described_class.process_start_time).to be_an(Float)
end
end
end