Refactored Sidekiq Throttler and updated documentation
This commit is contained in:
parent
208530494e
commit
e840749b84
|
@ -19,7 +19,7 @@ class ApplicationSetting < ActiveRecord::Base
|
|||
serialize :domain_whitelist, Array
|
||||
serialize :domain_blacklist, Array
|
||||
serialize :repository_storages
|
||||
serialize :sidekiq_throttling_queues
|
||||
serialize :sidekiq_throttling_queues, Array
|
||||
|
||||
cache_markdown_field :sign_in_text
|
||||
cache_markdown_field :help_page_text
|
||||
|
|
|
@ -304,9 +304,9 @@
|
|||
.form-group
|
||||
= f.label :sidekiq_throttling_factor, 'Throttling Factor', class: 'control-label col-sm-2'
|
||||
.col-sm-10
|
||||
= f.number_field :sidekiq_throttling_factor, class: 'form-control', min: '0', max: '0.99', step: '0.01'
|
||||
= f.number_field :sidekiq_throttling_factor, class: 'form-control', min: '0.01', max: '0.99', step: '0.01'
|
||||
.help-block
|
||||
The factor by which the queues should be throttled. A value between 0.1 and 0.9.
|
||||
The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.
|
||||
|
||||
%fieldset
|
||||
%legend Spam and Anti-bot Protection
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
---
|
||||
title: Added ability to throttle Sidekiq Jobs
|
||||
merge_request: 7292
|
||||
author: Patricio Cano
|
||||
author:
|
||||
|
|
|
@ -3,14 +3,15 @@
|
|||
> Note: Introduced with GitLab 8.14
|
||||
|
||||
When your GitLab installation needs to handle tens of thousands of background
|
||||
jobs, it can be convenient to prioritize queues that need to be executed
|
||||
immediately, e.g. user initiated actions like merging a Merge Request.
|
||||
jobs, it can be convenient to throttle queues that do not need to be executed
|
||||
immediately, e.g. long running jobs like Pipelines, thus allowing jobs that do
|
||||
need to be executed immediately to have access to more resources.
|
||||
|
||||
In order to accomplish this, you can limit the amount of workers that certain
|
||||
slow running queues get can have available. This is what we call Sidekiq Job
|
||||
slow running queues can have available. This is what we call Sidekiq Job
|
||||
Throttling. Depending on your infrastructure, you might have different slow
|
||||
running queues, which is why you can choose which queues to throttle and by
|
||||
how much you want to throttle them.
|
||||
running queues, which is why you can choose which queues you want to throttle
|
||||
and by how much you want to throttle them.
|
||||
|
||||
These settings are available in the Application Settings of your GitLab
|
||||
installation.
|
||||
|
@ -24,8 +25,8 @@ and rounded up to the closest full integer.
|
|||
So, for example, you set the `:concurrency` to 25 and the `Throttling factor` to
|
||||
0.1, the maximum workers assigned to the selected queues would be 3.
|
||||
|
||||
```
|
||||
limit = (factor * Sidekiq.options[:concurrency]).ceil
|
||||
```ruby
|
||||
queue_limit = (factor * Sidekiq.options[:concurrency]).ceil
|
||||
```
|
||||
|
||||
After enabling the job throttling, you will need to restart your GitLab
|
||||
|
|
|
@ -3,18 +3,20 @@ module Gitlab
|
|||
class << self
|
||||
def execute!
|
||||
if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
|
||||
current_application_settings.sidekiq_throttling_queues.each do |queue|
|
||||
Sidekiq::Queue[queue].limit = set_limit
|
||||
Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
|
||||
Sidekiq::Queue[queue].limit = queue_limit
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_limit
|
||||
factor = current_application_settings.sidekiq_throttling_factor
|
||||
|
||||
(factor * Sidekiq.options[:concurrency]).ceil
|
||||
def queue_limit
|
||||
@queue_limit ||=
|
||||
begin
|
||||
factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor
|
||||
(factor * Sidekiq.options[:concurrency]).ceil
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -11,12 +11,6 @@ describe Gitlab::SidekiqThrottler do
|
|||
)
|
||||
end
|
||||
|
||||
describe '#set_limit' do
|
||||
it 'returns the correct limit' do
|
||||
expect(Gitlab::SidekiqThrottler.send(:set_limit)).to eq 4
|
||||
end
|
||||
end
|
||||
|
||||
describe '#execute!' do
|
||||
it 'sets limits on the selected queues' do
|
||||
Gitlab::SidekiqThrottler.execute!
|
||||
|
|
Loading…
Reference in New Issue