Add latest changes from gitlab-org/gitlab@master
This commit is contained in:
parent
bee11af56d
commit
535d167d40
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
title: Add web_exporter to expose Prometheus metrics
|
||||||
|
merge_request: 17943
|
||||||
|
author:
|
||||||
|
type: added
|
|
@ -1017,7 +1017,14 @@ production: &base
|
||||||
sidekiq_exporter:
|
sidekiq_exporter:
|
||||||
# enabled: true
|
# enabled: true
|
||||||
# address: localhost
|
# address: localhost
|
||||||
# port: 3807
|
# port: 8082
|
||||||
|
|
||||||
|
# Web exporter is webserver built in to Unicorn/Puma to expose Prometheus metrics
|
||||||
|
# It runs alongside the `/metrics` endpoints to ease the publish of metrics
|
||||||
|
web_exporter:
|
||||||
|
# enabled: true
|
||||||
|
# address: localhost
|
||||||
|
# port: 8083
|
||||||
|
|
||||||
## Prometheus settings
|
## Prometheus settings
|
||||||
# Do not modify these settings here. They should be modified in /etc/gitlab/gitlab.rb
|
# Do not modify these settings here. They should be modified in /etc/gitlab/gitlab.rb
|
||||||
|
@ -1061,6 +1068,21 @@ production: &base
|
||||||
development:
|
development:
|
||||||
<<: *base
|
<<: *base
|
||||||
|
|
||||||
|
# We want to run web/sidekiq exporters for devs
|
||||||
|
# to catch errors from using them.
|
||||||
|
#
|
||||||
|
# We use random port to not block ability to run
|
||||||
|
# multiple instances of the service
|
||||||
|
monitoring:
|
||||||
|
sidekiq_exporter:
|
||||||
|
enabled: true
|
||||||
|
address: 127.0.0.1
|
||||||
|
port: 0
|
||||||
|
web_exporter:
|
||||||
|
enabled: true
|
||||||
|
address: 127.0.0.1
|
||||||
|
port: 0
|
||||||
|
|
||||||
test:
|
test:
|
||||||
<<: *base
|
<<: *base
|
||||||
gravatar:
|
gravatar:
|
||||||
|
|
|
@ -663,7 +663,11 @@ Settings.monitoring['ruby_sampler_interval'] ||= 60
|
||||||
Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
|
Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
|
||||||
Settings.monitoring.sidekiq_exporter['enabled'] ||= false
|
Settings.monitoring.sidekiq_exporter['enabled'] ||= false
|
||||||
Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
|
Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
|
||||||
Settings.monitoring.sidekiq_exporter['port'] ||= 3807
|
Settings.monitoring.sidekiq_exporter['port'] ||= 8082
|
||||||
|
Settings.monitoring['web_exporter'] ||= Settingslogic.new({})
|
||||||
|
Settings.monitoring.web_exporter['enabled'] ||= false
|
||||||
|
Settings.monitoring.web_exporter['address'] ||= 'localhost'
|
||||||
|
Settings.monitoring.web_exporter['port'] ||= 8083
|
||||||
|
|
||||||
#
|
#
|
||||||
# Testing settings
|
# Testing settings
|
||||||
|
|
|
@ -35,7 +35,7 @@ Sidekiq.configure_server do |config|
|
||||||
# webserver metrics are cleaned up in config.ru: `warmup` block
|
# webserver metrics are cleaned up in config.ru: `warmup` block
|
||||||
Prometheus::CleanupMultiprocDirService.new.execute
|
Prometheus::CleanupMultiprocDirService.new.execute
|
||||||
|
|
||||||
Gitlab::Metrics::SidekiqMetricsExporter.instance.start
|
Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -58,3 +58,24 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
|
||||||
Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
|
Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
if defined?(::Unicorn) || defined?(::Puma)
|
||||||
|
Gitlab::Cluster::LifecycleEvents.on_master_start do
|
||||||
|
Gitlab::Metrics::Exporter::WebExporter.instance.start
|
||||||
|
end
|
||||||
|
|
||||||
|
Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
|
||||||
|
# We need to ensure that before we re-exec server
|
||||||
|
# we do stop the exporter
|
||||||
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
||||||
|
end
|
||||||
|
|
||||||
|
Gitlab::Cluster::LifecycleEvents.on_worker_start do
|
||||||
|
# The `#close_on_exec=` takes effect only on `execve`
|
||||||
|
# but this does not happen for Ruby fork
|
||||||
|
#
|
||||||
|
# This does stop server, as it is running on master.
|
||||||
|
# However, ensures that we close the TCPSocket.
|
||||||
|
Gitlab::Metrics::Exporter::WebExporter.instance.stop
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
|
@ -45,7 +45,7 @@ require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
|
||||||
|
|
||||||
on_restart do
|
on_restart do
|
||||||
# Signal application hooks that we're about to restart
|
# Signal application hooks that we're about to restart
|
||||||
Gitlab::Cluster::LifecycleEvents.do_master_restart
|
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
|
||||||
end
|
end
|
||||||
|
|
||||||
before_fork do
|
before_fork do
|
||||||
|
|
|
@ -40,7 +40,7 @@ require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initial
|
||||||
|
|
||||||
on_restart do
|
on_restart do
|
||||||
# Signal application hooks that we're about to restart
|
# Signal application hooks that we're about to restart
|
||||||
Gitlab::Cluster::LifecycleEvents.do_master_restart
|
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
|
||||||
end
|
end
|
||||||
|
|
||||||
before_fork do
|
before_fork do
|
||||||
|
|
|
@ -85,7 +85,7 @@ require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
|
||||||
|
|
||||||
before_exec do |server|
|
before_exec do |server|
|
||||||
# Signal application hooks that we're about to restart
|
# Signal application hooks that we're about to restart
|
||||||
Gitlab::Cluster::LifecycleEvents.do_master_restart
|
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
|
||||||
end
|
end
|
||||||
|
|
||||||
run_once = true
|
run_once = true
|
||||||
|
|
|
@ -18,7 +18,7 @@ require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
|
||||||
|
|
||||||
before_exec do |server|
|
before_exec do |server|
|
||||||
# Signal application hooks that we're about to restart
|
# Signal application hooks that we're about to restart
|
||||||
Gitlab::Cluster::LifecycleEvents.do_master_restart
|
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
|
||||||
end
|
end
|
||||||
|
|
||||||
run_once = true
|
run_once = true
|
||||||
|
|
|
@ -1,8 +1,5 @@
|
||||||
# Geo database replication **(PREMIUM ONLY)**
|
# Geo database replication **(PREMIUM ONLY)**
|
||||||
|
|
||||||
NOTE: **Note:**
|
|
||||||
The following steps are for Omnibus installs only. Using Geo with source-based installs was **deprecated** in GitLab 11.5.
|
|
||||||
|
|
||||||
NOTE: **Note:**
|
NOTE: **Note:**
|
||||||
If your GitLab installation uses external (not managed by Omnibus) PostgreSQL
|
If your GitLab installation uses external (not managed by Omnibus) PostgreSQL
|
||||||
instances, the Omnibus roles will not be able to perform all necessary
|
instances, the Omnibus roles will not be able to perform all necessary
|
||||||
|
|
|
@ -327,14 +327,20 @@ is populated whenever `gitlab-ctl reconfigure` is run manually or as part of an
|
||||||
Reconfigure logs files are named according to the UNIX timestamp of when the reconfigure
|
Reconfigure logs files are named according to the UNIX timestamp of when the reconfigure
|
||||||
was initiated, such as `1509705644.log`
|
was initiated, such as `1509705644.log`
|
||||||
|
|
||||||
## `sidekiq_exporter.log`
|
## `sidekiq_exporter.log` and `web_exporter.log`
|
||||||
|
|
||||||
If Prometheus metrics and the Sidekiq Exporter are both enabled, Sidekiq will
|
If Prometheus metrics and the Sidekiq Exporter are both enabled, Sidekiq will
|
||||||
start a Web server and listen to the defined port (default: 3807). Access logs
|
start a Web server and listen to the defined port (default: 8082). Access logs
|
||||||
will be generated in `/var/log/gitlab/gitlab-rails/sidekiq_exporter.log` for
|
will be generated in `/var/log/gitlab/gitlab-rails/sidekiq_exporter.log` for
|
||||||
Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq_exporter.log` for
|
Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq_exporter.log` for
|
||||||
installations from source.
|
installations from source.
|
||||||
|
|
||||||
|
If Prometheus metrics and the Web Exporter are both enabled, Unicorn/Puma will
|
||||||
|
start a Web server and listen to the defined port (default: 8083). Access logs
|
||||||
|
will be generated in `/var/log/gitlab/gitlab-rails/web_exporter.log` for
|
||||||
|
Omnibus GitLab packages or in `/home/git/gitlab/log/web_exporter.log` for
|
||||||
|
installations from source.
|
||||||
|
|
||||||
[repocheck]: repository_checks.md
|
[repocheck]: repository_checks.md
|
||||||
[Rack Attack]: ../security/rack_attack.md
|
[Rack Attack]: ../security/rack_attack.md
|
||||||
[Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md
|
[Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md
|
||||||
|
|
|
@ -414,6 +414,42 @@ Get Flowdock service settings for a project.
|
||||||
GET /projects/:id/services/flowdock
|
GET /projects/:id/services/flowdock
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## GitHub **(PREMIUM)**
|
||||||
|
|
||||||
|
Code collaboration software.
|
||||||
|
|
||||||
|
### Create/Edit GitHub service
|
||||||
|
|
||||||
|
Set GitHub service for a project.
|
||||||
|
|
||||||
|
```
|
||||||
|
PUT /projects/:id/services/github
|
||||||
|
```
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
|
||||||
|
| Parameter | Type | Required | Description |
|
||||||
|
| --------- | ---- | -------- | ----------- |
|
||||||
|
| `token` | string | true | GitHub API token with `repo:status` OAuth scope |
|
||||||
|
| `repository_url` | string | true | GitHub repository URL |
|
||||||
|
| `static_context` | boolean | false | Append instance name instead of branch to [status check name](../user/project/integrations/github.md#static--dynamic-status-check-names) |
|
||||||
|
|
||||||
|
### Delete GitHub service
|
||||||
|
|
||||||
|
Delete GitHub service for a project.
|
||||||
|
|
||||||
|
```
|
||||||
|
DELETE /projects/:id/services/github
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get GitHub service settings
|
||||||
|
|
||||||
|
Get GitHub service settings for a project.
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /projects/:id/services/github
|
||||||
|
```
|
||||||
|
|
||||||
## Hangouts Chat
|
## Hangouts Chat
|
||||||
|
|
||||||
Google GSuite team collaboration tool.
|
Google GSuite team collaboration tool.
|
||||||
|
|
|
@ -40,7 +40,7 @@ module Gitlab
|
||||||
(@before_fork_hooks ||= []) << block
|
(@before_fork_hooks ||= []) << block
|
||||||
end
|
end
|
||||||
|
|
||||||
def on_master_restart(&block)
|
def on_before_master_restart(&block)
|
||||||
return unless in_clustered_environment?
|
return unless in_clustered_environment?
|
||||||
|
|
||||||
# Defer block execution
|
# Defer block execution
|
||||||
|
@ -70,12 +70,15 @@ module Gitlab
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def do_master_restart
|
def do_before_master_restart
|
||||||
@master_restart_hooks && @master_restart_hooks.each do |block|
|
@master_restart_hooks && @master_restart_hooks.each do |block|
|
||||||
block.call
|
block.call
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# DEPRECATED
|
||||||
|
alias_method :do_master_restart, :do_before_master_restart
|
||||||
|
|
||||||
# Puma doesn't use singletons (which is good) but
|
# Puma doesn't use singletons (which is good) but
|
||||||
# this means we need to pass through whether the
|
# this means we need to pass through whether the
|
||||||
# puma server is running in single mode or cluster mode
|
# puma server is running in single mode or cluster mode
|
||||||
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module Metrics
|
||||||
|
module Exporter
|
||||||
|
class BaseExporter < Daemon
|
||||||
|
attr_reader :server
|
||||||
|
|
||||||
|
def enabled?
|
||||||
|
settings.enabled
|
||||||
|
end
|
||||||
|
|
||||||
|
def settings
|
||||||
|
raise NotImplementedError
|
||||||
|
end
|
||||||
|
|
||||||
|
def log_filename
|
||||||
|
raise NotImplementedError
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def start_working
|
||||||
|
logger = WEBrick::Log.new(log_filename)
|
||||||
|
logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]"
|
||||||
|
|
||||||
|
access_log = [
|
||||||
|
[logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
|
||||||
|
]
|
||||||
|
|
||||||
|
@server = ::WEBrick::HTTPServer.new(
|
||||||
|
Port: settings.port, BindAddress: settings.address,
|
||||||
|
Logger: logger, AccessLog: access_log)
|
||||||
|
server.mount "/", Rack::Handler::WEBrick, rack_app
|
||||||
|
server.start
|
||||||
|
end
|
||||||
|
|
||||||
|
def stop_working
|
||||||
|
if server # rubocop:disable Cop/LineBreakAroundConditionalBlock
|
||||||
|
server.shutdown
|
||||||
|
server.listeners.each(&:close)
|
||||||
|
end
|
||||||
|
@server = nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def rack_app
|
||||||
|
Rack::Builder.app do
|
||||||
|
use Rack::Deflater
|
||||||
|
use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present?
|
||||||
|
run -> (env) { [404, {}, ['']] }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,20 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'webrick'
|
||||||
|
require 'prometheus/client/rack/exporter'
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module Metrics
|
||||||
|
module Exporter
|
||||||
|
class SidekiqExporter < BaseExporter
|
||||||
|
def settings
|
||||||
|
Settings.monitoring.sidekiq_exporter
|
||||||
|
end
|
||||||
|
|
||||||
|
def log_filename
|
||||||
|
File.join(Rails.root, 'log', 'sidekiq_exporter.log')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,20 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'webrick'
|
||||||
|
require 'prometheus/client/rack/exporter'
|
||||||
|
|
||||||
|
module Gitlab
|
||||||
|
module Metrics
|
||||||
|
module Exporter
|
||||||
|
class WebExporter < BaseExporter
|
||||||
|
def settings
|
||||||
|
Settings.monitoring.web_exporter
|
||||||
|
end
|
||||||
|
|
||||||
|
def log_filename
|
||||||
|
File.join(Rails.root, 'log', 'web_exporter.log')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,49 +0,0 @@
|
||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
require 'webrick'
|
|
||||||
require 'prometheus/client/rack/exporter'
|
|
||||||
|
|
||||||
module Gitlab
|
|
||||||
module Metrics
|
|
||||||
class SidekiqMetricsExporter < Daemon
|
|
||||||
LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log')
|
|
||||||
|
|
||||||
def enabled?
|
|
||||||
::Gitlab::Metrics.metrics_folder_present? && settings.enabled
|
|
||||||
end
|
|
||||||
|
|
||||||
def settings
|
|
||||||
Settings.monitoring.sidekiq_exporter
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
attr_reader :server
|
|
||||||
|
|
||||||
def start_working
|
|
||||||
logger = WEBrick::Log.new(LOG_FILENAME)
|
|
||||||
access_log = [
|
|
||||||
[logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
|
|
||||||
]
|
|
||||||
|
|
||||||
@server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address,
|
|
||||||
Logger: logger, AccessLog: access_log)
|
|
||||||
server.mount "/", Rack::Handler::WEBrick, rack_app
|
|
||||||
server.start
|
|
||||||
end
|
|
||||||
|
|
||||||
def stop_working
|
|
||||||
server.shutdown if server
|
|
||||||
@server = nil
|
|
||||||
end
|
|
||||||
|
|
||||||
def rack_app
|
|
||||||
Rack::Builder.app do
|
|
||||||
use Rack::Deflater
|
|
||||||
use ::Prometheus::Client::Rack::Exporter
|
|
||||||
run -> (env) { [404, {}, ['']] }
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -2,20 +2,29 @@
|
||||||
|
|
||||||
require 'spec_helper'
|
require 'spec_helper'
|
||||||
|
|
||||||
describe Gitlab::Metrics::SidekiqMetricsExporter do
|
describe Gitlab::Metrics::Exporter::BaseExporter do
|
||||||
let(:exporter) { described_class.new }
|
let(:exporter) { described_class.new }
|
||||||
let(:server) { double('server') }
|
let(:server) { double('server') }
|
||||||
|
let(:socket) { double('socket') }
|
||||||
|
let(:log_filename) { File.join(Rails.root, 'log', 'sidekiq_exporter.log') }
|
||||||
|
let(:settings) { double('settings') }
|
||||||
|
|
||||||
before do
|
before do
|
||||||
allow(::WEBrick::HTTPServer).to receive(:new).and_return(server)
|
allow(::WEBrick::HTTPServer).to receive(:new).and_return(server)
|
||||||
allow(server).to receive(:mount)
|
allow(server).to receive(:mount)
|
||||||
allow(server).to receive(:start)
|
allow(server).to receive(:start)
|
||||||
allow(server).to receive(:shutdown)
|
allow(server).to receive(:shutdown)
|
||||||
|
allow(server).to receive(:listeners) { [socket] }
|
||||||
|
allow(socket).to receive(:close)
|
||||||
|
allow_any_instance_of(described_class).to receive(:log_filename).and_return(log_filename)
|
||||||
|
allow_any_instance_of(described_class).to receive(:settings).and_return(settings)
|
||||||
end
|
end
|
||||||
|
|
||||||
describe 'when exporter is enabled' do
|
describe 'when exporter is enabled' do
|
||||||
before do
|
before do
|
||||||
allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(true)
|
allow(settings).to receive(:enabled).and_return(true)
|
||||||
|
allow(settings).to receive(:port).and_return(3707)
|
||||||
|
allow(settings).to receive(:address).and_return('localhost')
|
||||||
end
|
end
|
||||||
|
|
||||||
describe 'when exporter is stopped' do
|
describe 'when exporter is stopped' do
|
||||||
|
@ -31,8 +40,8 @@ describe Gitlab::Metrics::SidekiqMetricsExporter do
|
||||||
let(:address) { 'sidekiq_exporter_address' }
|
let(:address) { 'sidekiq_exporter_address' }
|
||||||
|
|
||||||
before do
|
before do
|
||||||
allow(Settings.monitoring.sidekiq_exporter).to receive(:port).and_return(port)
|
allow(settings).to receive(:port).and_return(port)
|
||||||
allow(Settings.monitoring.sidekiq_exporter).to receive(:address).and_return(address)
|
allow(settings).to receive(:address).and_return(address)
|
||||||
end
|
end
|
||||||
|
|
||||||
it 'starts server with port and address from settings' do
|
it 'starts server with port and address from settings' do
|
||||||
|
@ -74,6 +83,7 @@ describe Gitlab::Metrics::SidekiqMetricsExporter do
|
||||||
it 'shutdowns server' do
|
it 'shutdowns server' do
|
||||||
expect { exporter.stop }.to change { exporter.thread? }.from(true).to(false)
|
expect { exporter.stop }.to change { exporter.thread? }.from(true).to(false)
|
||||||
|
|
||||||
|
expect(socket).to have_received(:close)
|
||||||
expect(server).to have_received(:shutdown)
|
expect(server).to have_received(:shutdown)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -82,7 +92,7 @@ describe Gitlab::Metrics::SidekiqMetricsExporter do
|
||||||
|
|
||||||
describe 'when exporter is disabled' do
|
describe 'when exporter is disabled' do
|
||||||
before do
|
before do
|
||||||
allow(Settings.monitoring.sidekiq_exporter).to receive(:enabled).and_return(false)
|
allow(settings).to receive(:enabled).and_return(false)
|
||||||
end
|
end
|
||||||
|
|
||||||
describe '#start' do
|
describe '#start' do
|
Loading…
Reference in New Issue