diff --git a/lib/sidekiq/middleware/server/retry_jobs.rb b/lib/sidekiq/middleware/server/retry_jobs.rb
index 484a10a5..46bac825 100644
--- a/lib/sidekiq/middleware/server/retry_jobs.rb
+++ b/lib/sidekiq/middleware/server/retry_jobs.rb
@@ -111,13 +111,19 @@ module Sidekiq
 
         # delayed_job uses the same basic formula
         def seconds_to_delay(worker, count)
+          default_retry_in = (count ** 4) + 15 + (rand(30)*(count+1))
+
           if worker.sidekiq_retry_with?
-            worker.sidekiq_retry_with.call(count)
+            begin
+              worker.sidekiq_retry_with.call(count)
+            rescue Exception => e
+              logger.error { "Failure scheduling retry using the defined `sidekiq_retry_in`! #{e.message} "}
+              default_retry_in
+            end
           else
-            (count ** 4) + 15 + (rand(30)*(count+1))
+            default_retry_in
           end
         end
-
       end
     end
   end
diff --git a/test/test_retry.rb b/test/test_retry.rb
index 8633d348..e29d5723 100644
--- a/test/test_retry.rb
+++ b/test/test_retry.rb
@@ -205,6 +205,17 @@ class TestRetry < Minitest::Test
     end
 
     describe "custom retry delay" do
+      before do
+        @old_logger    = Sidekiq.logger
+        @tmp_log_path  = '/tmp/sidekiq-retries.log'
+        Sidekiq.logger = Logger.new(@tmp_log_path)
+      end
+
+      after do
+        Sidekiq.logger = @old_logger
+        Sidekiq.options.delete(:logfile)
+        File.unlink @tmp_log_path if File.exists?(@tmp_log_path)
+      end
 
       let(:custom_worker) do
         Class.new do
@@ -216,6 +227,16 @@ class TestRetry < Minitest::Test
         end
       end
 
+      let(:error_worker) do
+        Class.new do
+          include ::Sidekiq::Worker
+
+          sidekiq_retry_in do |count|
+            count / 0
+          end
+        end
+      end
+
       let(:handler){ Sidekiq::Middleware::Server::RetryJobs.new }
 
       it "retries with a default delay" do
@@ -225,6 +246,12 @@ class TestRetry < Minitest::Test
       it "retries with a custom delay" do
         assert_equal 4, handler.seconds_to_delay(custom_worker, 2)
       end
+
+      it "falls back to the default retry on exception" do
+        refute_equal 4, handler.seconds_to_delay(error_worker, 2)
+        assert_match(/Failure scheduling retry using the defined `sidekiq_retry_in`/,
+                     File.read(@tmp_log_path), 'Log entry missing for sidekiq_retry_in')
+      end
     end
   end