merge master

6.0-Upgrade.md

@@ -0,0 +1,47 @@
+# Welcome to Sidekiq 6.0!
+
+Sidekiq 6.0 contains some breaking changes which streamline proper operation
+of Sidekiq.  It also drops support for EOL versions of Ruby and Rails.
+
+## What's New
+
+This release has major breaking changes.  Read and test carefully in production.
+
+- Include **new JSON log format** for use where you want machine-ingestible log output. Enable with:
+```
+Sidekiq.configure_server do |config|
+  config.logger_format = :json # or :default
+end
+```
+- **Remove the daemonization, logfile and pidfile command line arguments**.
+  I've [noted for years](https://www.mikeperham.com/2014/09/22/dont-daemonize-your-daemons/)
+ how modern services should be managed with a proper init system.
+Managing services manually is more error-prone, let your operating system do it for you.
+- **Validate proper usage of the `REDIS_PROVIDER` variable.**
+  This variable is meant to hold the name of the environment
+variable which contains your Redis URL, so that you can switch Redis
+providers quickly and easily with a single variable change.  It is not
+meant to hold the actual Redis URL itself.  If you want to manually set
+the Redis URL then you may set `REDIS_URL` directly. [#3969]
+- **Increase default shutdown timeout from 8 seconds to 25 seconds.**
+  Both Heroku and ECS now use 30 second shutdown timeout
+by default and we want Sidekiq to take advantage of this time.  If you
+have deployment scripts which depend on the old default timeout, use `-t 8` to
+get the old behavior. [#3968]
+* **Rails <5** is no longer supported.
+* **Ruby <2.4** is no longer supported.
+
+## Upgrade
+
+As always, please upgrade Sidekiq **one major version at a time**.
+If you are already running Sidekiq 5.x, then:
+
+* Upgrade to the latest Sidekiq 5.x.
+```ruby
+gem 'sidekiq', '< 6'
+```
+* Fix any deprecation warnings you see.
+* Upgrade to 6.x.
+```ruby
+gem 'sidekiq', '< 7'
+```

Pro-Changes.md

@@ -7,6 +7,7 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
 HEAD
 ---------
 
+- Better handling of invalid job JSON by reliable scheduler [#4053]
 - Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
 
 4.0.4

lib/sidekiq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'sidekiq/version'
-fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2')
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.4.0." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.4.0')
 
 require 'sidekiq/logging'
 require 'sidekiq/client'
@@ -96,8 +96,8 @@ module Sidekiq
       begin
         yield conn
       rescue Redis::CommandError => ex
-        #2550 Failover can cause the server to become a slave, need
-        # to disconnect and reopen the socket to get back to the master.
+        #2550 Failover can cause the server to become a replica, need
+        # to disconnect and reopen the socket to get back to the primary.
         (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
         raise
       end
@@ -158,12 +158,6 @@ module Sidekiq
     defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
   end
 
-  def self.default_retries_exhausted=(prok)
-    logger.info { "default_retries_exhausted is deprecated, please use `config.death_handlers << -> {|job, ex| }`" }
-    return nil unless prok
-    death_handlers << prok
-  end
-
   ##
   # Death handlers are called when all retries for a job have been exhausted and
   # the job dies.  It's the notification to your application

lib/sidekiq/cli.rb

@@ -27,7 +27,7 @@ module Sidekiq
     attr_accessor :launcher
     attr_accessor :environment
 
-    def parse(args=ARGV)
+    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
@@ -194,17 +194,33 @@ module Sidekiq
     alias_method :☠, :exit
 
     def setup_options(args)
+      # parse CLI options
       opts = parse_options(args)
+
       set_environment opts[:environment]
 
-      options[:queues] << 'default' if options[:queues].empty?
+      # check config file presence
+      if opts[:config_file]
+        if opts[:config_file] && !File.exist?(opts[:config_file])
+          raise ArgumentError, "No such file #{opts[:config_file]}"
+        end
+      else
+        if opts[:require] && File.directory?(opts[:require])
+          %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
+            path = File.expand_path(filename, opts[:require])
+            opts[:config_file] ||= path if File.exist?(path)
+          end
+        end
+      end
 
-      cfile = opts[:config_file]
-      opts = parse_config(cfile).merge(opts) if cfile
+      # parse config file options
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
+      opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
-      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
+      # merge with defaults
       options.merge!(opts)
     end
 
@@ -325,11 +341,8 @@ module Sidekiq
         logger.info @parser
         die 1
       end
-      @parser.parse!(argv)
 
-      %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
-        opts[:config_file] ||= filename if File.exist?(filename)
-      end
+      @parser.parse!(argv)
 
       opts
     end
@@ -344,19 +357,17 @@ module Sidekiq
       opts = {}
       if File.exist?(cfile)
         opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
-
-        if opts.respond_to? :deep_symbolize_keys!
-          opts.deep_symbolize_keys!
-        else
-          symbolize_keys_deep!(opts)
-        end
-
-        opts = opts.merge(opts.delete(environment.to_sym) || {})
-        parse_queues(opts, opts.delete(:queues) || [])
-      else
-        # allow a non-existent config file so Sidekiq
-        # can be deployed by cap with just the defaults.
       end
+
+      if opts.respond_to? :deep_symbolize_keys!
+        opts.deep_symbolize_keys!
+      else
+        symbolize_keys_deep!(opts)
+      end
+
+      opts = opts.merge(opts.delete(environment.to_sym) || {})
+      parse_queues(opts, opts.delete(:queues) || [])
+
       ns = opts.delete(:namespace)
       if ns
         # logger hasn't been initialized yet, puts is all we have.
@@ -370,10 +381,10 @@ module Sidekiq
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end
 
-    def parse_queue(opts, q, weight=nil)
+    def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
-      raise ArgumentError, "queues: #{q} cannot be defined twice" if opts[:queues].include?(q)
-      [weight.to_i, 1].max.times { opts[:queues] << q }
+      raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
+      [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
   end

test/dummy/config/sidekiq.yml

@@ -0,0 +1,2 @@
+---
+:concurrency: 25

test/test_cli.rb

@@ -203,6 +203,17 @@ class TestCLI < Minitest::Test
             assert_equal 3, Sidekiq.options[:queues].count { |q| q == 'seldom' }
           end
         end
+
+        describe 'default config file' do
+          describe 'when required path is a directory' do
+            it 'tries config/sidekiq.yml' do
+              @cli.parse(%w[sidekiq -r ./test/dummy])
+
+              assert_equal 'sidekiq.yml', File.basename(Sidekiq.options[:config_file])
+              assert_equal 25, Sidekiq.options[:concurrency]
+            end
+          end
+        end
       end
     end
 
@@ -219,6 +230,14 @@ class TestCLI < Minitest::Test
           exit = assert_raises(SystemExit) { @cli.parse(%w[sidekiq -r ./test/fixtures]) }
           assert_equal 1, exit.status
         end
+
+        describe 'when config file path does not exist' do
+          it 'raises argument error' do
+            assert_raises(ArgumentError) do
+              @cli.parse(%w[sidekiq -r ./test/fake_env.rb -C /non/existent/path])
+            end
+          end
+        end
       end
     end
   end