f0c52df5e5
Sidekiq jobs frequently spawn long-lived child processes to do work. In some circumstances, these can be reparented to init when sidekiq is terminated, leading to duplication of work and strange concurrency problems. This commit changes sidekiq so that, if run as a process group leader, it will forward `INT` and `TERM` signals to the whole process group. If the memory killer is active, it will also use the process group when resorting to `kill -9` to shut down. These changes mean that a naive `kill <pid-of-sidekiq>` will now do the right thing, killing any child processes spawned by sidekiq, as long as the process supervisor placed it in its own process group. If sidekiq isn't a process group leader, this new code is skipped.
92 lines
3.6 KiB
Ruby
92 lines
3.6 KiB
Ruby
require 'sidekiq/web'
|
|
|
|
# Disable the Sidekiq Rack session since GitLab already has its own session store.
|
|
# CSRF protection still works (https://github.com/mperham/sidekiq/commit/315504e766c4fd88a29b7772169060afc4c40329).
|
|
Sidekiq::Web.set :sessions, false
|
|
|
|
# Custom Queues configuration
|
|
queues_config_hash = Gitlab::Redis::Queues.params
|
|
queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
|
|
|
|
# Default is to retry 25 times with exponential backoff. That's too much.
|
|
Sidekiq.default_worker_options = { retry: 3 }
|
|
|
|
if Rails.env.development?
|
|
Sidekiq.default_worker_options[:backtrace] = true
|
|
end
|
|
|
|
enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
|
|
|
|
Sidekiq.configure_server do |config|
|
|
config.redis = queues_config_hash
|
|
|
|
config.server_middleware do |chain|
|
|
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs
|
|
chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
|
|
chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
|
|
chain.add Gitlab::SidekiqMiddleware::BatchLoader
|
|
chain.add Gitlab::SidekiqMiddleware::CorrelationLogger
|
|
chain.add Gitlab::SidekiqStatus::ServerMiddleware
|
|
end
|
|
|
|
if enable_json_logs
|
|
Sidekiq.logger.formatter = Gitlab::SidekiqLogging::JSONFormatter.new
|
|
config.options[:job_logger] = Gitlab::SidekiqLogging::StructuredLogger
|
|
end
|
|
|
|
config.client_middleware do |chain|
|
|
chain.add Gitlab::SidekiqStatus::ClientMiddleware
|
|
chain.add Gitlab::SidekiqMiddleware::CorrelationInjector
|
|
end
|
|
|
|
config.on :startup do
|
|
# Clear any connections that might have been obtained before starting
|
|
# Sidekiq (e.g. in an initializer).
|
|
ActiveRecord::Base.clear_all_connections!
|
|
end
|
|
|
|
if Feature::FlipperFeature.table_exists? && Feature.enabled?(:gitlab_sidekiq_reliable_fetcher)
|
|
# By default we're going to use Semi Reliable Fetch
|
|
config.options[:semi_reliable_fetch] = Feature.enabled?(:gitlab_sidekiq_enable_semi_reliable_fetcher, default_enabled: true)
|
|
Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
|
|
end
|
|
|
|
# Sidekiq-cron: load recurring jobs from gitlab.yml
|
|
# UGLY Hack to get nested hash from settingslogic
|
|
cron_jobs = JSON.parse(Gitlab.config.cron_jobs.to_json)
|
|
# UGLY hack: Settingslogic doesn't allow 'class' key
|
|
cron_jobs_required_keys = %w(job_class cron)
|
|
cron_jobs.each do |k, v|
|
|
if cron_jobs[k] && cron_jobs_required_keys.all? { |s| cron_jobs[k].key?(s) }
|
|
cron_jobs[k]['class'] = cron_jobs[k].delete('job_class')
|
|
else
|
|
cron_jobs.delete(k)
|
|
Rails.logger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.")
|
|
end
|
|
end
|
|
Sidekiq::Cron::Job.load_from_hash! cron_jobs
|
|
|
|
Gitlab::SidekiqVersioning.install!
|
|
|
|
db_config = Gitlab::Database.config ||
|
|
Rails.application.config.database_configuration[Rails.env]
|
|
db_config['pool'] = Sidekiq.options[:concurrency]
|
|
ActiveRecord::Base.establish_connection(db_config)
|
|
Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}")
|
|
|
|
# Avoid autoload issue such as 'Mail::Parsers::AddressStruct'
|
|
# https://github.com/mikel/mail/issues/912#issuecomment-214850355
|
|
Mail.eager_autoload!
|
|
|
|
# Ensure the whole process group is terminated if possible
|
|
Gitlab::SidekiqSignals.install!(Sidekiq::CLI::SIGNAL_HANDLERS)
|
|
end
|
|
|
|
Sidekiq.configure_client do |config|
|
|
config.redis = queues_config_hash
|
|
|
|
config.client_middleware do |chain|
|
|
chain.add Gitlab::SidekiqMiddleware::CorrelationInjector
|
|
chain.add Gitlab::SidekiqStatus::ClientMiddleware
|
|
end
|
|
end
|