Adds new metrics for unicorn monitoring

This adds new metrics for monitoring unicorn. These metrics include
process_cpu_seconds_total, process_start_time_seconds, process_max_fds,
and unicorn_workers.

Gemfile

@@ -406,6 +406,7 @@ gem 'health_check', '~> 2.6.0'
 # System information
 gem 'vmstat', '~> 2.3.0'
 gem 'sys-filesystem', '~> 1.1.6'
+gem 'sys-proctable', '~> 1.2'
 
 # SSH host key support
 gem 'net-ssh', '~> 5.0'

Gemfile.lock

@@ -867,6 +867,8 @@ GEM
       state_machines-activemodel (>= 0.5.0)
     sys-filesystem (1.1.6)
       ffi
+    sys-proctable (1.2.1)
+      ffi
     sysexits (1.2.0)
     temple (0.8.0)
     test-prof (0.2.5)
@@ -1160,6 +1162,7 @@ DEPENDENCIES
   stackprof (~> 0.2.10)
   state_machines-activerecord (~> 0.5.1)
   sys-filesystem (~> 1.1.6)
+  sys-proctable (~> 1.2)
   test-prof (~> 0.2.5)
   thin (~> 1.7.0)
   timecop (~> 0.8.0)

lib/gitlab/metrics/samplers/unicorn_sampler.rb

@@ -8,12 +8,19 @@ module Gitlab
           super(interval)
         end
 
-        def unicorn_active_connections
+        def metrics
-          @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
+          @metrics ||= init_metrics
         end
 
-        def unicorn_queued_connections
+        def init_metrics
-          @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
+          {
+            unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max),
+            unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max),
+            unicorn_workers:            ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers'),
+            process_cpu_seconds_total:  ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'),
+            process_max_fds:            ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'),
+            process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds')
+          }
         end
 
         def enabled?
@@ -23,14 +30,19 @@ module Gitlab
 
         def sample
           Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active)
+            metrics[:unicorn_active_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued)
+            metrics[:unicorn_queued_connections].set({ socket_type: 'tcp', socket_address: addr }, stats.queued)
           end
 
           Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
-            unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active)
+            metrics[:unicorn_active_connections].set({ socket_type: 'unix', socket_address: addr }, stats.active)
-            unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued)
+            metrics[:unicorn_queued_connections].set({ socket_type: 'unix', socket_address: addr }, stats.queued)
           end
+
+          metrics[:process_cpu_seconds_total].set({ pid: nil }, ::Gitlab::Metrics::System.cpu_time)
+          metrics[:process_start_time_seconds].set({ pid: nil }, ::Gitlab::Metrics::System.process_start_time)
+          metrics[:process_max_fds].set({ pid: nil }, ::Gitlab::Metrics::System.max_open_file_descriptors)
+          metrics[:unicorn_workers].set({}, unicorn_workers_count)
         end
 
         private
@@ -39,6 +51,10 @@ module Gitlab
           @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
         end
 
+        def pid
+          @pid ||= Process.pid
+        end
+
         def unix_listeners
           @unix_listeners ||= Unicorn.listener_names - tcp_listeners
         end
@@ -46,6 +62,10 @@ module Gitlab
         def unicorn_with_listeners?
           defined?(Unicorn) && Unicorn.listener_names.any?
         end
+
+        def unicorn_workers_count
+          Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker/)}.count
+        end
       end
     end
   end

lib/gitlab/metrics/system.rb

@@ -23,6 +23,16 @@ module Gitlab
         def self.file_descriptor_count
           Dir.glob('/proc/self/fd/*').length
         end
+
+        def self.max_open_file_descriptors
+          match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/)
+
+          if match && match[1]
+            max_fds = match[1].to_i
+          end
+
+          max_fds
+        end
       else
         def self.memory_usage
           0.0
@@ -31,6 +41,10 @@ module Gitlab
         def self.file_descriptor_count
           0
         end
+
+        def self.max_open_file_descriptors
+          0
+        end
       end
 
       # THREAD_CPUTIME is not supported on OS X
@@ -46,6 +60,17 @@ module Gitlab
         end
       end
 
+      # CLOCK_BOOTTIME is not supported on OS X
+      if Process.const_defined?(:CLOCK_BOOTTIME)
+        def self.process_start_time
+          Process
+            .clock_gettime(Process::CLOCK_BOOTTIME, :float_second)
+        end
+      else
+        def self.process_start_time
+          0.0
+        end
+      end
       # Returns the current real time in a given precision.
       #
       # Returns the time as a Float for precision = :float_second.

spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb

@@ -39,8 +39,8 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
         it 'updates metrics type unix and with addr' do
           labels = { socket_type: 'unix', socket_address: socket_address }
 
-          expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+          expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
-          expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
+          expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
 
           subject.sample
         end
@@ -50,7 +50,6 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
     context 'unicorn listens on tcp sockets' do
       let(:tcp_socket_address) { '0.0.0.0:8080' }
       let(:tcp_sockets) { [tcp_socket_address] }
-
       before do
         allow(unicorn).to receive(:listener_names).and_return(tcp_sockets)
       end
@@ -71,13 +70,37 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
         it 'updates metrics type unix and with addr' do
           labels = { socket_type: 'tcp', socket_address: tcp_socket_address }
 
-          expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+          expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
-          expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
+          expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
 
           subject.sample
         end
       end
     end
+
+    context 'additional metrics' do
+      let(:cpu_time) { 3.14 }
+      let(:process_start_time) { 19100.24 }
+      let(:process_max_fds) { 1024 }
+      let(:unicorn_workers) { 2 }
+
+      before do
+        allow(unicorn).to receive(:listener_names).and_return([""])
+        allow(::Gitlab::Metrics::System).to receive(:cpu_time).and_return(cpu_time)
+        allow(::Gitlab::Metrics::System).to receive(:process_start_time).and_return(process_start_time)
+        allow(::Gitlab::Metrics::System).to receive(:max_open_file_descriptors).and_return(process_max_fds)
+        allow(subject).to receive(:unicorn_workers_count).and_return(unicorn_workers)
+      end
+
+      it "sets additional metrics" do
+        expect(subject.metrics[:process_cpu_seconds_total]).to receive(:set).with({ pid: nil }, cpu_time)
+        expect(subject.metrics[:process_start_time_seconds]).to receive(:set).with({ pid: nil }, process_start_time)
+        expect(subject.metrics[:process_max_fds]).to receive(:set).with({ pid: nil }, process_max_fds)
+        expect(subject.metrics[:unicorn_workers]).to receive(:set).with({}, unicorn_workers)
+
+        subject.sample
+      end
+    end
   end
 
   describe '#start' do

spec/lib/gitlab/metrics/system_spec.rb

@@ -13,6 +13,12 @@ describe Gitlab::Metrics::System do
         expect(described_class.file_descriptor_count).to be > 0
       end
     end
+
+    describe '.max_open_file_descriptors' do
+      it 'returns the max allowed open file descriptors' do
+        expect(described_class.max_open_file_descriptors).to be > 0
+      end
+    end
   else
     describe '.memory_usage' do
       it 'returns 0.0' do
@@ -25,6 +31,12 @@ describe Gitlab::Metrics::System do
         expect(described_class.file_descriptor_count).to eq(0)
       end
     end
+
+    describe '.max_open_file_descriptors' do
+      it 'returns 0' do
+        expect(described_class.max_open_file_descriptors).to eq(0)
+      end
+    end
   end
 
   describe '.cpu_time' do
@@ -44,4 +56,10 @@ describe Gitlab::Metrics::System do
       expect(described_class.monotonic_time).to be_an(Float)
     end
   end
+
+  describe '.process_start_time' do
+    it 'returns a Float' do
+      expect(described_class.process_start_time).to be_an(Float)
+    end
+  end
 end