Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2019-12-22 09:07:51 +00:00
parent 62fcb9ffa9
commit 3ad11f24ca
38 changed files with 230 additions and 71 deletions

View File

@ -163,7 +163,7 @@ module Git
end
def logger
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Sidekiq.logger
else
# This service runs in Sidekiq, so this shouldn't ever be

View File

@ -22,6 +22,7 @@ module Gitlab
require_dependency Rails.root.join('lib/gitlab/current_settings')
require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')
require_dependency Rails.root.join('lib/gitlab/runtime')
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
@ -255,7 +256,7 @@ module Gitlab
caching_config_hash[:compress] = false
caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
if Sidekiq.server? || defined?(::Puma) # threaded context
if Gitlab::Runtime.multi_threaded?
caching_config_hash[:pool_size] = Gitlab::Redis::Cache.pool_size
caching_config_hash[:pool_timeout] = 1
end

View File

@ -46,7 +46,7 @@ Rails.application.configure do
# Do not log asset requests
config.assets.quiet = true
config.allow_concurrency = defined?(::Puma)
config.allow_concurrency = Gitlab::Runtime.multi_threaded?
# BetterErrors live shell (REPL) on every stack frame
BetterErrors::Middleware.allow_ip!("127.0.0.1/0")

View File

@ -75,5 +75,5 @@ Rails.application.configure do
config.eager_load = true
config.allow_concurrency = defined?(::Puma)
config.allow_concurrency = Gitlab::Runtime.multi_threaded?
end

View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
begin
current_runtime = Gitlab::Runtime.identify
Gitlab::AppLogger.info("Process #{Process.pid} (#{$0}) identified as: #{current_runtime}")
rescue => e
message = <<-NOTICE
\n!! RUNTIME IDENTIFICATION FAILED: #{e}
Runtime based configuration settings may not work properly.
If you continue to see this error, please file an issue via
https://gitlab.com/gitlab-org/gitlab/issues/new
NOTICE
Gitlab::AppLogger.error(message)
Gitlab::ErrorTracking.track_exception(e)
end

View File

@ -364,7 +364,7 @@ Gitlab.ee do
# To ensure acceptable performance we only allow feature to be used with
# multithreaded web-server Puma. This will be removed once download logic is moved
# to GitLab workhorse
Settings.dependency_proxy['enabled'] = false unless defined?(::Puma)
Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma?
end
#

View File

@ -4,11 +4,11 @@ require 'prometheus/client'
def prometheus_default_multiproc_dir
return unless Rails.env.development? || Rails.env.test?
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
elsif defined?(Unicorn::Worker)
elsif Gitlab::Runtime.unicorn?
Rails.root.join('tmp/prometheus_multiproc_dir/unicorn')
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
else
Rails.root.join('tmp/prometheus_multiproc_dir')
@ -48,9 +48,9 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
Gitlab::Cluster::LifecycleEvents.on_master_start do
::Prometheus::Client.reinitialize_on_pid_change(force: true)
if defined?(::Unicorn)
if Gitlab::Runtime.unicorn?
Gitlab::Metrics::Samplers::UnicornSampler.instance(Settings.monitoring.unicorn_sampler_interval).start
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
Gitlab::Metrics::Samplers::PumaSampler.instance(Settings.monitoring.puma_sampler_interval).start
end
@ -58,7 +58,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
end
end
if defined?(::Unicorn) || defined?(::Puma)
if Gitlab::Runtime.web_server?
Gitlab::Cluster::LifecycleEvents.on_master_start do
Gitlab::Metrics::Exporter::WebExporter.instance.start
end

View File

@ -2,7 +2,7 @@
# Don't handle sidekiq configuration as it
# has its own special active record configuration here
if defined?(ActiveRecord::Base) && !Sidekiq.server?
if defined?(ActiveRecord::Base) && !Gitlab::Runtime.sidekiq?
Gitlab::Cluster::LifecycleEvents.on_worker_start do
ActiveSupport.on_load(:active_record) do
ActiveRecord::Base.establish_connection

View File

@ -5,10 +5,8 @@
#
# Follow-up the issue: https://gitlab.com/gitlab-org/gitlab/issues/34107
if defined?(::Puma)
if Gitlab::Runtime.puma?
Puma::Cluster.prepend(::Gitlab::Cluster::Mixins::PumaCluster)
end
if defined?(::Unicorn::HttpServer)
elsif Gitlab::Runtime.unicorn?
Unicorn::HttpServer.prepend(::Gitlab::Cluster::Mixins::UnicornHttpServer)
end

View File

@ -2,7 +2,7 @@
# when running on puma, scale connection pool size with the number
# of threads per worker process
if defined?(::Puma)
if Gitlab::Runtime.puma?
db_config = Gitlab::Database.config ||
Rails.application.config.database_configuration[Rails.env]
puma_options = Puma.cli_config.options

View File

@ -1,5 +1,5 @@
# Only use Lograge for Rails
unless Sidekiq.server?
unless Gitlab::Runtime.sidekiq?
filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
Rails.application.configure do

View File

@ -9,7 +9,7 @@
# and it's used only as the last resort. In such case this termination is
# logged and we should fix the potential timeout issue in the code itself.
if defined?(::Puma) && !Rails.env.test?
if Gitlab::Runtime.puma? && !Rails.env.test?
require 'rack/timeout/base'
Gitlab::Application.configure do |config|

View File

@ -13,7 +13,7 @@ if Labkit::Tracing.enabled?
end
# Instrument Sidekiq server calls when running Sidekiq server
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Sidekiq.configure_server do |config|
config.server_middleware do |chain|
chain.add Labkit::Tracing::Sidekiq::ServerMiddleware

View File

@ -1,5 +1,5 @@
# frozen_string_literal: true
if defined?(::Puma) && ::Puma.cli_config.options[:workers].to_i.zero?
if Gitlab::Runtime.puma? && ::Puma.cli_config.options[:workers].to_i.zero?
raise 'Puma is only supported in Cluster-mode: workers > 0'
end

View File

@ -100,8 +100,8 @@ module Gitlab
end
def self.process_name
return 'sidekiq' if Sidekiq.server?
return 'console' if defined?(Rails::Console)
return 'sidekiq' if Gitlab::Runtime.sidekiq?
return 'console' if Gitlab::Runtime.console?
return 'test' if Rails.env.test?
'web'

View File

@ -149,10 +149,10 @@ module Gitlab
def in_clustered_environment?
# Sidekiq doesn't fork
return false if Sidekiq.server?
return false if Gitlab::Runtime.sidekiq?
# Unicorn always forks
return true if defined?(::Unicorn)
return true if Gitlab::Runtime.unicorn?
# Puma sometimes forks
return true if in_clustered_puma?
@ -162,7 +162,7 @@ module Gitlab
end
def in_clustered_puma?
return false unless defined?(::Puma)
return false unless Gitlab::Runtime.puma?
@puma_options && @puma_options[:workers] && @puma_options[:workers] > 0
end

View File

@ -29,7 +29,7 @@ module Gitlab
PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 30
CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = '.gitaly-metadata'
MUTEX = Mutex.new
@ -382,17 +382,13 @@ module Gitlab
end
def self.long_timeout
if web_app_server?
if Gitlab::Runtime.web_server?
default_timeout
else
6.hours
end
end
def self.web_app_server?
defined?(::Unicorn) || defined?(::Puma)
end
def self.storage_metadata_file_path(storage)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
File.join(

View File

@ -135,7 +135,7 @@ module Gitlab
end
def cleanup_time
Sidekiq.server? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S
Gitlab::Runtime.sidekiq? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S
end
def tmp_keychains_created

View File

@ -18,7 +18,7 @@ module Gitlab
end
def check
return unless defined?(::Puma)
return unless Gitlab::Runtime.puma?
stats = Puma.stats
stats = JSON.parse(stats)

View File

@ -30,7 +30,7 @@ module Gitlab
# to change so we can cache the list of servers.
def http_servers
strong_memoize(:http_servers) do
next unless defined?(::Unicorn::HttpServer)
next unless Gitlab::Runtime.unicorn?
ObjectSpace.each_object(::Unicorn::HttpServer).to_a
end

View File

@ -68,7 +68,7 @@ module Gitlab
end
def timeout_time
Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
Gitlab::Runtime.sidekiq? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
end
def link_dependencies(text, highlighted_text)

View File

@ -150,7 +150,7 @@ module Gitlab
# Returns the prefix to use for the name of a series.
def series_prefix
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
@series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
end
# Allow access from other metrics related middlewares

View File

@ -39,14 +39,10 @@ module Gitlab
end
def add_metric(series, values, tags = {})
prefix = sidekiq? ? 'sidekiq_' : 'rails_'
prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
@metrics << Metric.new("#{prefix}#{series}", values, tags)
end
def sidekiq?
Sidekiq.server?
end
end
end
end

View File

@ -61,7 +61,7 @@ module Gitlab
# it takes around 80ms. The instances of HttpServers are not a subject
# to change so we can cache the list of servers.
def http_servers
return [] unless defined?(::Unicorn::HttpServer)
return [] unless Gitlab::Runtime.unicorn?
@http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a
end

View File

@ -22,10 +22,10 @@ module Gitlab
def pool_size
# heuristic constant 5 should be a config setting somewhere -- related to CPU count?
size = 5
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
# the pool will be used in a multi-threaded context
size += Sidekiq.options[:concurrency]
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
size += Puma.cli_config.options[:max_threads]
end

57
lib/gitlab/runtime.rb Normal file
View File

@ -0,0 +1,57 @@
# frozen_string_literal: true
module Gitlab
# Provides routines to identify the current runtime as which the application
# executes, such as whether it is an application server and which one.
module Runtime
AmbiguousProcessError = Class.new(StandardError)
UnknownProcessError = Class.new(StandardError)
class << self
def identify
matches = []
matches << :puma if puma?
matches << :unicorn if unicorn?
matches << :console if console?
matches << :sidekiq if sidekiq?
if matches.one?
matches.first
elsif matches.none?
raise UnknownProcessError.new(
"Failed to identify runtime for process #{Process.pid} (#{$0})"
)
else
raise AmbiguousProcessError.new(
"Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})"
)
end
end
def puma?
!!defined?(::Puma)
end
# For unicorn, we need to check for actual server instances to avoid false positives.
def unicorn?
!!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer))
end
def sidekiq?
!!(defined?(::Sidekiq) && Sidekiq.server?)
end
def console?
!!defined?(::Rails::Console)
end
def web_server?
puma? || unicorn?
end
def multi_threaded?
puma? || sidekiq?
end
end
end
end

View File

@ -5,11 +5,11 @@ module Prometheus
extend self
def worker_id
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
sidekiq_worker_id
elsif defined?(Unicorn::Worker)
elsif Gitlab::Runtime.unicorn?
unicorn_worker_id
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
puma_worker_id
else
unknown_process_id

View File

@ -16,6 +16,7 @@ describe 'Database config initializer' do
let(:puma_options) { { max_threads: 8 } }
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
stub_const("Puma", puma)
allow(puma).to receive_message_chain(:cli_config, :options).and_return(puma_options)
end

View File

@ -26,7 +26,7 @@ describe Gitlab::GitalyClient do
context 'running in Unicorn' do
before do
stub_const('Unicorn', 1)
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
end
it { expect(subject.long_timeout).to eq(55) }
@ -34,7 +34,7 @@ describe Gitlab::GitalyClient do
context 'running in Puma' do
before do
stub_const('Puma', 1)
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
end
it { expect(subject.long_timeout).to eq(55) }

View File

@ -236,7 +236,7 @@ describe Gitlab::Gpg do
context 'when running in Sidekiq' do
before do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
end
it_behaves_like 'multiple deletion attempts of the tmp-dir', described_class::BG_CLEANUP_RUNTIME_S

View File

@ -22,6 +22,7 @@ describe Gitlab::HealthChecks::PumaCheck do
context 'when Puma is not loaded' do
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(false)
hide_const('Puma')
end
@ -33,6 +34,7 @@ describe Gitlab::HealthChecks::PumaCheck do
context 'when Puma is loaded' do
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
stub_const('Puma', Module.new)
end

View File

@ -26,6 +26,7 @@ describe Gitlab::HealthChecks::UnicornCheck do
context 'when Unicorn is not loaded' do
before do
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(false)
hide_const('Unicorn')
end
@ -39,6 +40,7 @@ describe Gitlab::HealthChecks::UnicornCheck do
let(:http_server_class) { Struct.new(:worker_processes) }
before do
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
stub_const('Unicorn::HttpServer', http_server_class)
end

View File

@ -111,7 +111,7 @@ describe Gitlab::Highlight do
end
it 'utilizes longer timeout for sidekiq' do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Timeout).to receive(:timeout).with(described_class::TIMEOUT_BACKGROUND).and_call_original
subject.highlight("Content")

View File

@ -63,7 +63,7 @@ describe Gitlab::Metrics::Samplers::InfluxSampler do
describe '#add_metric' do
it 'prefixes the series name for a Rails process' do
expect(sampler).to receive(:sidekiq?).and_return(false)
expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
expect(Gitlab::Metrics::Metric).to receive(:new)
.with('rails_cats', { value: 10 }, {})
@ -73,7 +73,7 @@ describe Gitlab::Metrics::Samplers::InfluxSampler do
end
it 'prefixes the series name for a Sidekiq process' do
expect(sampler).to receive(:sidekiq?).and_return(true)
expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Gitlab::Metrics::Metric).to receive(:new)
.with('sidekiq_cats', { value: 10 }, {})

View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Runtime do
context "when unknown" do
it "raises an exception when trying to identify" do
expect { subject.identify }.to raise_error(subject::UnknownProcessError)
end
end
context "on multiple matches" do
before do
stub_const('::Puma', double)
stub_const('::Rails::Console', double)
end
it "raises an exception when trying to identify" do
expect { subject.identify }.to raise_error(subject::AmbiguousProcessError)
end
end
context "puma" do
let(:puma_type) { double('::Puma') }
before do
stub_const('::Puma', puma_type)
end
it "identifies itself" do
expect(subject.identify).to eq(:puma)
expect(subject.puma?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.console?).to be(false)
end
end
context "unicorn" do
let(:unicorn_type) { Module.new }
let(:unicorn_server_type) { Class.new }
before do
stub_const('::Unicorn', unicorn_type)
stub_const('::Unicorn::HttpServer', unicorn_server_type)
end
it "identifies itself" do
expect(subject.identify).to eq(:unicorn)
expect(subject.unicorn?).to be(true)
end
it "does not identify as others" do
expect(subject.puma?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.console?).to be(false)
end
end
context "sidekiq" do
let(:sidekiq_type) { double('::Sidekiq') }
before do
stub_const('::Sidekiq', sidekiq_type)
allow(sidekiq_type).to receive(:server?).and_return(true)
end
it "identifies itself" do
expect(subject.identify).to eq(:sidekiq)
expect(subject.sidekiq?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.puma?).to be(false)
expect(subject.console?).to be(false)
end
end
context "console" do
let(:console_type) { double('::Rails::Console') }
before do
stub_const('::Rails::Console', console_type)
end
it "identifies itself" do
expect(subject.identify).to eq(:console)
expect(subject.console?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.puma?).to be(false)
end
end
end

View File

@ -6,16 +6,13 @@ describe Prometheus::PidProvider do
describe '.worker_id' do
subject { described_class.worker_id }
let(:sidekiq_module) { Module.new }
before do
allow(sidekiq_module).to receive(:server?).and_return(false)
stub_const('Sidekiq', sidekiq_module)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
end
context 'when running in Sidekiq server mode' do
before do
expect(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
end
context 'in a clustered setup' do
@ -33,8 +30,7 @@ describe Prometheus::PidProvider do
context 'when running in Unicorn mode' do
before do
stub_const('Unicorn::Worker', Class.new)
hide_const('Puma')
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
expect(described_class).to receive(:process_name)
.at_least(:once)
@ -94,8 +90,7 @@ describe Prometheus::PidProvider do
context 'when running in Puma mode' do
before do
stub_const('Puma', Module.new)
hide_const('Unicorn::Worker')
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
expect(described_class).to receive(:process_name)
.at_least(:once)
@ -116,11 +111,6 @@ describe Prometheus::PidProvider do
end
context 'when running in unknown mode' do
before do
hide_const('Puma')
hide_const('Unicorn::Worker')
end
it { is_expected.to eq "process_#{Process.pid}" }
end
end

View File

@ -108,7 +108,7 @@ describe Git::BranchPushService, services: true do
end
it 'reports an error' do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Sidekiq.logger).to receive(:warn)
expect { subject }.not_to change { Ci::Pipeline.count }

View File

@ -118,7 +118,7 @@ RSpec.shared_examples "redis_shared_examples" do
context 'when running not on sidekiq workers' do
before do
allow(Sidekiq).to receive(:server?).and_return(false)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
end
it 'instantiates a connection pool with size 5' do
@ -130,7 +130,7 @@ RSpec.shared_examples "redis_shared_examples" do
context 'when running on sidekiq workers' do
before do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 })
end