2019-03-30 07:15:48 +00:00
# frozen_string_literal: true
2016-10-21 16:13:41 +00:00
require 'spec_helper'
2020-06-24 15:08:50 +00:00
RSpec . describe 'Every Sidekiq worker' do
2020-01-22 12:08:40 +00:00
let ( :workers_without_defaults ) do
2021-03-16 12:11:31 +00:00
Gitlab :: SidekiqConfig . workers - Gitlab :: SidekiqConfig :: DEFAULT_WORKERS . values
2020-01-22 12:08:40 +00:00
end
2016-10-21 16:13:41 +00:00
it 'does not use the default queue' do
2020-01-22 12:08:40 +00:00
expect ( workers_without_defaults . map ( & :queue ) ) . not_to include ( 'default' )
2016-10-21 16:13:41 +00:00
end
it 'uses the cronjob queue when the worker runs as a cronjob' do
2017-11-28 16:16:50 +00:00
expect ( Gitlab :: SidekiqConfig . cron_workers . map ( & :queue ) ) . to all ( start_with ( 'cronjob:' ) )
2016-10-21 16:13:41 +00:00
end
2018-11-19 21:29:19 +00:00
it 'has its queue in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS' , :aggregate_failures do
2017-11-28 16:16:50 +00:00
file_worker_queues = Gitlab :: SidekiqConfig . worker_queues . to_set
2016-10-21 16:13:41 +00:00
2017-11-28 16:16:50 +00:00
worker_queues = Gitlab :: SidekiqConfig . workers . map ( & :queue ) . to_set
2020-07-14 12:09:14 +00:00
worker_queues << ActionMailer :: MailDeliveryJob . new . queue_name
2017-11-28 16:16:50 +00:00
worker_queues << 'default'
missing_from_file = worker_queues - file_worker_queues
2018-11-19 21:29:19 +00:00
expect ( missing_from_file ) . to be_empty , " expected #{ missing_from_file . to_a . inspect } to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS "
2017-11-28 16:16:50 +00:00
2019-10-30 15:14:17 +00:00
unnecessarily_in_file = file_worker_queues - worker_queues
expect ( unnecessarily_in_file ) . to be_empty , " expected #{ unnecessarily_in_file . to_a . inspect } not to be in Gitlab::SidekiqConfig::QUEUE_CONFIG_PATHS "
2017-11-28 16:16:50 +00:00
end
it 'has its queue or namespace in config/sidekiq_queues.yml' , :aggregate_failures do
config_queues = Gitlab :: SidekiqConfig . config_queues . to_set
Gitlab :: SidekiqConfig . workers . each do | worker |
queue = worker . queue
queue_namespace = queue . split ( ':' ) . first
expect ( config_queues ) . to include ( queue ) . or ( include ( queue_namespace ) )
end
2016-10-21 16:13:41 +00:00
end
2019-10-18 11:11:44 +00:00
2020-06-12 12:08:56 +00:00
it 'has a value for loggable_arguments' do
workers_without_defaults . each do | worker |
expect ( worker . klass . loggable_arguments ) . to be_an ( Array )
end
end
2019-10-18 11:11:44 +00:00
describe " feature category declarations " do
let ( :feature_categories ) do
YAML . load_file ( Rails . root . join ( 'config' , 'feature_categories.yml' ) ) . map ( & :to_sym ) . to_set
end
# All Sidekiq worker classes should declare a valid `feature_category`
2019-10-30 15:14:17 +00:00
# or explicitly be excluded with the `feature_category_not_owned!` annotation.
2020-07-08 21:09:09 +00:00
# Please see doc/development/sidekiq_style_guide.md#feature-categorization for more details.
2019-10-18 11:11:44 +00:00
it 'has a feature_category or feature_category_not_owned! attribute' , :aggregate_failures do
2020-01-22 12:08:40 +00:00
workers_without_defaults . each do | worker |
2019-10-18 11:11:44 +00:00
expect ( worker . get_feature_category ) . to be_a ( Symbol ) , " expected #{ worker . inspect } to declare a feature_category or feature_category_not_owned! "
end
end
# All Sidekiq worker classes should declare a valid `feature_category`.
# The category should match a value in `config/feature_categories.yml`.
2020-07-08 21:09:09 +00:00
# Please see doc/development/sidekiq_style_guide.md#feature-categorization for more details.
2019-10-18 11:11:44 +00:00
it 'has a feature_category that maps to a value in feature_categories.yml' , :aggregate_failures do
2020-01-22 12:08:40 +00:00
workers_with_feature_categories = workers_without_defaults
2019-10-18 11:11:44 +00:00
. select ( & :get_feature_category )
. reject ( & :feature_category_not_owned? )
workers_with_feature_categories . each do | worker |
expect ( feature_categories ) . to include ( worker . get_feature_category ) , " expected #{ worker . inspect } to declare a valid feature_category, but got #{ worker . get_feature_category } "
end
end
2019-10-30 15:14:17 +00:00
# Memory-bound workers are very expensive to run, since they need to run on nodes with very low
# concurrency, so that each job can consume a large amounts of memory. For this reason, on
# GitLab.com, when a large number of memory-bound jobs arrive at once, we let them queue up
# rather than scaling the hardware to meet the SLO. For this reason, memory-bound,
2020-03-02 18:07:42 +00:00
# high urgency jobs are explicitly discouraged and disabled.
it 'is (exclusively) memory-bound or high urgency, not both' , :aggregate_failures do
high_urgency_workers = workers_without_defaults
. select { | worker | worker . get_urgency == :high }
2019-10-30 15:14:17 +00:00
2020-03-02 18:07:42 +00:00
high_urgency_workers . each do | worker |
expect ( worker . get_worker_resource_boundary ) . not_to eq ( :memory ) , " #{ worker . inspect } cannot be both memory-bound and high urgency "
2019-10-30 15:14:17 +00:00
end
end
2020-03-02 18:07:42 +00:00
# In high traffic installations, such as GitLab.com, `urgency :high` workers run in a
# dedicated fleet. In order to ensure short queue times, `urgency :high` jobs have strict
2019-10-30 15:14:17 +00:00
# SLOs in order to ensure throughput. However, when a worker depends on an external service,
# such as a user's k8s cluster or a third-party internet service, we cannot guarantee latency,
# and therefore throughput. An outage to an 3rd party service could therefore impact throughput
2020-03-02 18:07:42 +00:00
# on other high urgency jobs, leading to degradation through the GitLab application.
# Please see doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies for more
2019-10-30 15:14:17 +00:00
# details.
2020-03-02 18:07:42 +00:00
it 'has (exclusively) external dependencies or is high urgency, not both' , :aggregate_failures do
high_urgency_workers = workers_without_defaults
. select { | worker | worker . get_urgency == :high }
2019-10-30 15:14:17 +00:00
2020-03-02 18:07:42 +00:00
high_urgency_workers . each do | worker |
expect ( worker . worker_has_external_dependencies? ) . to be_falsey , " #{ worker . inspect } cannot have both external dependencies and be high urgency "
2019-10-30 15:14:17 +00:00
end
end
2019-10-18 11:11:44 +00:00
end
2016-10-21 16:13:41 +00:00
end