From 94ccd5410d84313be48dc949fdf1d185c98a9d7f Mon Sep 17 00:00:00 2001 From: Daniel Morton Date: Wed, 28 Jul 2021 18:32:19 -0400 Subject: [PATCH] `retry_on` parameter `attempts` now accepts `:unlimited` (#41761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In some applications, some classes of errors may be raised during the execution of a job which the developer would want to retry forever. These classes of errors would most likely be infrastructure problems that the developer knows will be resolved eventually but may take a variable amount of time or errors where due to application business logic, there could be something temporarily blocking the job from executing, like a resource that is needed for the job being locked for a lengthy amount of time. While an arbitrarily large number of attempts could previously be passed, this is inexpressive as sometimes the developer may just need the job to continue to be retried until it eventually succeeds. Without this, developers would need to include additional code to handle the situation where the job eventually fails its attempts limit and has to be re-enqueued manually. As with many things this should be used with caution and only for errors that the developer knows will definitely eventually be resolved, allowing the job to continue. [Daniel Morton + Rafael Mendonça França] --- activejob/CHANGELOG.md | 15 +++++++++++++++ activejob/lib/active_job/exceptions.rb | 6 ++++-- activejob/test/cases/exceptions_test.rb | 8 ++++++++ activejob/test/jobs/retry_job.rb | 2 ++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/activejob/CHANGELOG.md b/activejob/CHANGELOG.md index f3cae74672..220a69d988 100644 --- a/activejob/CHANGELOG.md +++ b/activejob/CHANGELOG.md @@ -1,3 +1,18 @@ +* Allow a job to retry indefinitely + + The `attempts` parameter of the `retry_on` method now accepts the + symbol reference `:unlimited` in addition to a specific number of retry + attempts to allow a developer to specify that a job should retry + forever until it succeeds. + + class MyJob < ActiveJob::Base + retry_on(AlwaysRetryException, attempts: :unlimited) + + # the actual job code + end + + *Daniel Morton* + * Added possibility to check on `:priority` in test helper methods `assert_enqueued_with` and `assert_performed_with`. diff --git a/activejob/lib/active_job/exceptions.rb b/activejob/lib/active_job/exceptions.rb index 9ff305dedc..2a0d42120d 100644 --- a/activejob/lib/active_job/exceptions.rb +++ b/activejob/lib/active_job/exceptions.rb @@ -25,7 +25,8 @@ module ActiveJob # as a computing proc that takes the number of executions so far as an argument, or as a symbol reference of # :exponentially_longer, which applies the wait algorithm of ((executions**4) + (Kernel.rand * (executions**4) * jitter)) + 2 # (first wait ~3s, then ~18s, then ~83s, etc) - # * :attempts - Re-enqueues the job the specified number of times (default: 5 attempts) + # * :attempts - Re-enqueues the job the specified number of times (default: 5 attempts) or a symbol reference of :unlimited + # to retry the job until it succeeds # * :queue - Re-enqueues the job on a different queue # * :priority - Re-enqueues the job with a different priority # * :jitter - A random delay of wait time used when calculating backoff. The default is 15% (0.15) which represents the upper bound of possible wait time (expressed as a percentage) @@ -35,6 +36,7 @@ module ActiveJob # class RemoteServiceJob < ActiveJob::Base # retry_on CustomAppException # defaults to ~3s wait, 5 attempts # retry_on AnotherCustomAppException, wait: ->(executions) { executions * 2 } + # retry_on CustomInfrastructureException, wait: 5.minutes, attempts: :unlimited # # retry_on ActiveRecord::Deadlocked, wait: 5.seconds, attempts: 3 # retry_on Net::OpenTimeout, Timeout::Error, wait: :exponentially_longer, attempts: 10 # retries at most 10 times for Net::OpenTimeout and Timeout::Error combined @@ -56,7 +58,7 @@ module ActiveJob def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: JITTER_DEFAULT) rescue_from(*exceptions) do |error| executions = executions_for(exceptions) - if executions < attempts + if attempts == :unlimited || executions < attempts retry_job wait: determine_delay(seconds_or_duration_or_algorithm: wait, executions: executions, jitter: jitter), queue: queue, priority: priority, error: error else if block_given? diff --git a/activejob/test/cases/exceptions_test.rb b/activejob/test/cases/exceptions_test.rb index 5555b51b0a..a018b998e9 100644 --- a/activejob/test/cases/exceptions_test.rb +++ b/activejob/test/cases/exceptions_test.rb @@ -300,6 +300,14 @@ class ExceptionsTest < ActiveSupport::TestCase assert_equal ["Raised ActiveJob::DeserializationError for the 5 time"], JobBuffer.values end + test "successfully retry job throwing UnlimitedRetryError a few times" do + RetryJob.perform_later "UnlimitedRetryError", 10 + + assert_equal 10, JobBuffer.values.size + assert_equal "Raised UnlimitedRetryError for the 9th time", JobBuffer.values[8] + assert_equal "Successfully completed job", JobBuffer.values[9] + end + test "running a job enqueued by AJ 5.2" do job = RetryJob.new("DefaultsError", 6) job.exception_executions = nil # This is how jobs from Rails 5.2 will look diff --git a/activejob/test/jobs/retry_job.rb b/activejob/test/jobs/retry_job.rb index 3dfc6f02be..8d8c93fa38 100644 --- a/activejob/test/jobs/retry_job.rb +++ b/activejob/test/jobs/retry_job.rb @@ -17,6 +17,7 @@ class DiscardableError < StandardError; end class FirstDiscardableErrorOfTwo < StandardError; end class SecondDiscardableErrorOfTwo < StandardError; end class CustomDiscardableError < StandardError; end +class UnlimitedRetryError < StandardError; end class RetryJob < ActiveJob::Base retry_on DefaultsError @@ -29,6 +30,7 @@ class RetryJob < ActiveJob::Base retry_on CustomWaitTenAttemptsError, wait: ->(executions) { executions * 2 }, attempts: 10 retry_on(CustomCatchError) { |job, error| JobBuffer.add("Dealt with a job that failed to retry in a custom way after #{job.arguments.second} attempts. Message: #{error.message}") } retry_on(ActiveJob::DeserializationError) { |job, error| JobBuffer.add("Raised #{error.class} for the #{job.executions} time") } + retry_on UnlimitedRetryError, attempts: :unlimited discard_on DiscardableError discard_on FirstDiscardableErrorOfTwo, SecondDiscardableErrorOfTwo