2019-10-18 07:11:44 -04:00
# frozen_string_literal: true
module WorkerAttributes
extend ActiveSupport :: Concern
2020-07-07 05:08:57 -04:00
include Gitlab :: ClassAttributes
2019-10-18 07:11:44 -04:00
2019-10-30 11:14:17 -04:00
# Resource boundaries that workers can declare through the
2020-03-02 13:07:42 -05:00
# `resource_boundary` attribute
2019-10-30 11:14:17 -04:00
VALID_RESOURCE_BOUNDARIES = [ :memory , :cpu , :unknown ] . freeze
2020-03-02 13:07:42 -05:00
# Urgencies that workers can declare through the `urgencies` attribute
2020-03-12 14:09:28 -04:00
VALID_URGENCIES = [ :high , :low , :throttled ] . freeze
2020-03-02 13:07:42 -05:00
2021-03-26 05:09:24 -04:00
VALID_DATA_CONSISTENCIES = [ :always , :sticky , :delayed ] . freeze
2020-01-24 13:09:00 -05:00
NAMESPACE_WEIGHTS = {
auto_devops : 2 ,
auto_merge : 3 ,
chaos : 2 ,
deployment : 3 ,
mail_scheduler : 2 ,
notifications : 2 ,
pipeline_cache : 3 ,
pipeline_creation : 4 ,
pipeline_default : 3 ,
pipeline_hooks : 2 ,
pipeline_processing : 5 ,
# EE-specific
epics : 2 ,
2020-02-05 16:09:02 -05:00
incident_management : 2 ,
security_scans : 2
2020-01-24 13:09:00 -05:00
} . stringify_keys . freeze
2019-10-18 07:11:44 -04:00
class_methods do
2020-07-07 05:08:57 -04:00
def feature_category ( value , * extras )
2019-10-18 07:11:44 -04:00
raise " Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned " if value == :not_owned
2021-04-28 08:10:09 -04:00
set_class_attribute ( :feature_category , value )
2019-10-18 07:11:44 -04:00
end
# Special case: mark this work as not associated with a feature category
# this should be used for cross-cutting concerns, such as mailer workers.
def feature_category_not_owned!
2021-04-28 08:10:09 -04:00
set_class_attribute ( :feature_category , :not_owned )
2019-10-18 07:11:44 -04:00
end
def get_feature_category
2020-07-07 05:08:57 -04:00
get_class_attribute ( :feature_category )
2019-10-18 07:11:44 -04:00
end
def feature_category_not_owned?
2020-07-07 05:08:57 -04:00
get_feature_category == :not_owned
2019-10-18 07:11:44 -04:00
end
2020-03-02 13:07:42 -05:00
# This should be set to :high for jobs that need to be run
# immediately, or, if they are delayed, risk creating
# inconsistencies in the application that could being perceived by
# the user as incorrect behavior (ie, a bug)
#
# See
# doc/development/sidekiq_style_guide.md#urgency
2019-10-30 11:14:17 -04:00
# for details
2020-03-02 13:07:42 -05:00
def urgency ( urgency )
raise " Invalid urgency: #{ urgency } " unless VALID_URGENCIES . include? ( urgency )
2021-04-28 08:10:09 -04:00
set_class_attribute ( :urgency , urgency )
2019-10-30 11:14:17 -04:00
end
2020-03-02 13:07:42 -05:00
def get_urgency
2020-07-07 05:08:57 -04:00
class_attributes [ :urgency ] || :low
2019-10-30 11:14:17 -04:00
end
2021-05-19 17:12:42 -04:00
# Allows configuring worker's data_consistency.
#
# Worker can utilize Sidekiq readonly database replicas capabilities by setting data_consistency attribute.
# Workers with data_consistency set to :delayed or :sticky, calling #perform_async
# will be delayed in order to give replication process enough time to complete.
#
# - *data_consistency* values:
# - 'always' - The job is required to use the primary database (default).
# - 'sticky' - The uses a replica as long as possible. It switches to primary either on write or long replication lag.
# - 'delayed' - The job would switch to primary only on write. It would use replica always.
# If there's a long replication lag the job will be delayed, and only if the replica is not up to date on the next retry,
# it will switch to the primary.
# - *feature_flag* - allows you to toggle a job's `data_consistency, which permits you to safely toggle load balancing capabilities for a specific job.
# If disabled, job will default to `:always`, which means that the job will always use the primary.
2021-03-26 05:09:24 -04:00
def data_consistency ( data_consistency , feature_flag : nil )
raise ArgumentError , " Invalid data consistency: #{ data_consistency } " unless VALID_DATA_CONSISTENCIES . include? ( data_consistency )
raise ArgumentError , 'Data consistency is already set' if class_attributes [ :data_consistency ]
2021-04-28 08:10:09 -04:00
set_class_attribute ( :data_consistency_feature_flag , feature_flag ) if feature_flag
set_class_attribute ( :data_consistency , data_consistency )
2021-03-26 05:09:24 -04:00
validate_worker_attributes!
end
def validate_worker_attributes!
# Since the deduplication should always take into account the latest binary replication pointer into account,
# not the first one, the deduplication will not work with sticky or delayed.
# Follow up issue to improve this: https://gitlab.com/gitlab-org/gitlab/-/issues/325291
2021-05-19 17:12:42 -04:00
if idempotent? && utilizes_load_balancing_capabilities?
2021-03-26 05:09:24 -04:00
raise ArgumentError , " Class can't be marked as idempotent if data_consistency is not set to :always "
end
end
2021-05-19 17:12:42 -04:00
# If data_consistency is not set to :always, worker will try to utilize load balancing capabilities and use the replica
def utilizes_load_balancing_capabilities?
get_data_consistency != :always
end
2021-03-26 05:09:24 -04:00
def get_data_consistency
class_attributes [ :data_consistency ] || :always
end
def get_data_consistency_feature_flag_enabled?
return true unless class_attributes [ :data_consistency_feature_flag ]
Feature . enabled? ( class_attributes [ :data_consistency_feature_flag ] , default_enabled : :yaml )
end
2019-10-30 11:14:17 -04:00
# Set this attribute on a job when it will call to services outside of the
# application, such as 3rd party applications, other k8s clusters etc See
2020-07-08 17:09:09 -04:00
# doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies for
2019-10-30 11:14:17 -04:00
# details
def worker_has_external_dependencies!
2021-04-28 08:10:09 -04:00
set_class_attribute ( :external_dependencies , true )
2019-10-30 11:14:17 -04:00
end
# Returns a truthy value if the worker has external dependencies.
2020-07-08 17:09:09 -04:00
# See doc/development/sidekiq_style_guide.md#jobs-with-external-dependencies
2019-10-30 11:14:17 -04:00
# for details
def worker_has_external_dependencies?
2020-07-07 05:08:57 -04:00
class_attributes [ :external_dependencies ]
2019-10-30 11:14:17 -04:00
end
def worker_resource_boundary ( boundary )
raise " Invalid boundary " unless VALID_RESOURCE_BOUNDARIES . include? boundary
2021-04-28 08:10:09 -04:00
set_class_attribute ( :resource_boundary , boundary )
2019-10-30 11:14:17 -04:00
end
def get_worker_resource_boundary
2020-07-07 05:08:57 -04:00
class_attributes [ :resource_boundary ] || :unknown
2019-10-30 11:14:17 -04:00
end
2020-02-19 13:09:10 -05:00
def idempotent!
2021-04-28 08:10:09 -04:00
set_class_attribute ( :idempotent , true )
2021-03-26 05:09:24 -04:00
validate_worker_attributes!
2020-02-19 13:09:10 -05:00
end
def idempotent?
2020-07-07 05:08:57 -04:00
class_attributes [ :idempotent ]
2020-02-19 13:09:10 -05:00
end
2020-01-24 13:09:00 -05:00
def weight ( value )
2021-04-28 08:10:09 -04:00
set_class_attribute ( :weight , value )
2020-01-24 13:09:00 -05:00
end
def get_weight
2020-07-07 05:08:57 -04:00
class_attributes [ :weight ] ||
2020-01-24 13:09:00 -05:00
NAMESPACE_WEIGHTS [ queue_namespace ] ||
1
end
2020-05-25 23:08:02 -04:00
def tags ( * values )
2021-04-28 08:10:09 -04:00
set_class_attribute ( :tags , values )
2020-05-25 23:08:02 -04:00
end
def get_tags
2020-07-07 05:08:57 -04:00
Array ( class_attributes [ :tags ] )
2020-05-25 23:08:02 -04:00
end
2020-06-16 20:16:40 -04:00
def deduplicate ( strategy , options = { } )
2021-04-28 08:10:09 -04:00
set_class_attribute ( :deduplication_strategy , strategy )
set_class_attribute ( :deduplication_options , options )
2020-06-16 20:16:40 -04:00
end
def get_deduplicate_strategy
2020-07-07 05:08:57 -04:00
class_attributes [ :deduplication_strategy ] ||
2020-06-16 20:16:40 -04:00
Gitlab :: SidekiqMiddleware :: DuplicateJobs :: DuplicateJob :: DEFAULT_STRATEGY
end
def get_deduplication_options
2020-07-07 05:08:57 -04:00
class_attributes [ :deduplication_options ] || { }
2019-10-18 07:11:44 -04:00
end
2021-02-25 07:11:01 -05:00
2021-06-04 11:10:25 -04:00
def deduplication_enabled?
return true unless get_deduplication_options [ :feature_flag ]
Feature . enabled? ( get_deduplication_options [ :feature_flag ] , default_enabled : :yaml )
end
2021-02-25 07:11:01 -05:00
def big_payload!
2021-04-28 08:10:09 -04:00
set_class_attribute ( :big_payload , true )
2021-02-25 07:11:01 -05:00
end
def big_payload?
class_attributes [ :big_payload ]
end
2019-10-18 07:11:44 -04:00
end
end