Pause hard shutdown to give time for ensure cleanup, fixes #5095

If we raised Sidekiq::Shutdown in job threads, sleep for a while
so they have time to run `ensure` blocks. Right now we immediately
return from hard_shutdown and call `exit` which immediately raises SystemExit on every thread which can stop ensure blocks.

Changes.md

@@ -25,6 +25,7 @@ SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
 bin/rails generate sidekiq:job ProcessOrderJob
 ```
 - Fix job retries losing CurrentAttributes [#5090]
+- Tweak shutdown to give long-running threads time to cleanup [#5095]
 
 6.3.1
 ---------

lib/sidekiq/manager.rb

@@ -55,9 +55,6 @@ module Sidekiq
       fire_event(:quiet, reverse: true)
     end
 
-    # hack for quicker development / testing environment #2774
-    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
-
     def stop(deadline)
       quiet
       fire_event(:shutdown, reverse: true)
@@ -69,12 +66,7 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      while remaining > PAUSE_TIME
-        return if @workers.empty?
-        sleep PAUSE_TIME
-        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      end
+      wait_for(deadline) { @workers.empty? }
       return if @workers.empty?
 
       hard_shutdown
@@ -130,6 +122,12 @@ module Sidekiq
       cleanup.each do |processor|
         processor.kill
       end
+
+      # when this method returns, we immediately call `exit` which may not give
+      # the remaining threads time to run `ensure` blocks, etc. We pause here up
+      # to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
+      wait_for(deadline) { @workers.empty? }
     end
   end
 end

lib/sidekiq/util.rb

@@ -39,6 +39,19 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex