merge master

.travis.yml

@@ -3,8 +3,6 @@ sudo: false
 cache: bundler
 services:
   - redis-server
-gemfile:
-  - gemfiles/rails_5.gemfile
 bundler_args: --without development load_test
 rvm:
   - 2.5.1

Appraisals

@@ -1,4 +0,0 @@
-appraise "rails-5" do
-  gem "rails", "~> 5.2"
-  gem 'activerecord-jdbcsqlite3-adapter', '>= 50', platforms: :jruby
-end

COMM-LICENSE

@@ -14,13 +14,15 @@ In order to use the Software under this Agreement, you must receive a “Source
 
 1.2 Unlimited Organization License. If you purchased an Organization License (included with the Sidekiq Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
 
-1.3 Limited Organization License. If you purchased an Organization License (included with the Sidekiq Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. 
+1.3 Limited Enterprise License. If you purchased a Limited License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. 
 
-1.4 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA.
+1.4 Enterprise Site License. If you purchased a Site License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
 
-1.5 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
+1.5 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA.
 
-1.6 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
+1.6 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
+
+1.7 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”).        
 
 2. Modifications. Contributed Systems shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.
 

Changes.md

@@ -2,7 +2,7 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-HEAD
+6.0
 ---------
 
 This release has major breaking changes.  Read and test carefully in production.
@@ -23,9 +23,16 @@ get the old behavior. [#3968]
   arguments to Sidekiq. Use a proper process supervisor (e.g. systemd or
 foreman) to manage Sidekiq.
 
+5.2.5
+---------
+
+- Fix default usage of `config/sidekiq.yml` [#4077, Tensho]
+
 5.2.4
 ---------
 
+- Add warnings for various deprecations and changes coming in Sidekiq 6.0.
+  See the 6-0 branch. [#4056]
 - Various improvements to the Sidekiq test suite and coverage [#4026, #4039, Tensho]
 
 5.2.3

Ent-Changes.md

@@ -7,6 +7,13 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
 HEAD
 -------------
 
+- Require Sidekiq Pro 4.0 and Sidekiq 5.2.
+- Refactor historical metrics API to use revamped Statsd support in Sidekiq Pro
+- Add a gauge to historical metrics for `default` queue latency [#4079]
+
+1.7.2
+-------------
+
 - Add PT and JA translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
 - Fix elapsed time calculations to use monotonic clock [#4000, sj26]
 - Fix edge case where flapping leadership would cause old periodic

Gemfile

@@ -5,18 +5,11 @@ gemspec
 gem 'rake'
 gem 'redis-namespace'
 gem 'rails', '~> 5.2'
-gem 'sqlite3', platforms: :ruby
+gem 'sqlite3', '~> 1.3.6', platforms: :ruby
 gem 'activerecord-jdbcsqlite3-adapter', platforms: :jruby
 
-group :development do
-  gem 'appraisal'
-end
-
 group :test do
   gem 'minitest'
-  gem 'minitest-around'
-  gem 'minitest-focus'
-  gem 'minitest-reporters'
   gem 'simplecov'
 end
 

Pro-Changes.md

@@ -7,6 +7,8 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
 HEAD
 ---------
 
+- Increase super\_fetch retriever thread count from 1 to 2 to make it
+  less sensitive to Redis latency.
 - Better handling of invalid job JSON by reliable scheduler [#4053]
 - Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
 

gemfiles/rails_5.gemfile

@@ -1,32 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "rake"
-gem "redis-namespace"
-gem "rails", "~> 5.2"
-gem "sqlite3", platforms: :ruby
-gem "activerecord-jdbcsqlite3-adapter", ">= 50", platforms: :jruby
-
-group :development do
-  gem "appraisal"
-end
-
-group :test do
-  gem "minitest"
-  gem "minitest-around"
-  gem "minitest-focus"
-  gem "minitest-reporters"
-  gem "simplecov"
-end
-
-group :development, :test do
-  gem "pry-byebug", platforms: :mri
-end
-
-group :load_test do
-  gem "hiredis"
-  gem "toxiproxy"
-end
-
-gemspec path: "../"

lib/sidekiq/cli.rb

@@ -62,6 +62,7 @@ module Sidekiq
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+      logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -197,17 +198,22 @@ module Sidekiq
           raise ArgumentError, "No such file #{opts[:config_file]}"
         end
       else
-        if opts[:require] && File.directory?(opts[:require])
-          %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
-            path = File.expand_path(filename, opts[:require])
-            opts[:config_file] ||= path if File.exist?(path)
-          end
+        config_dir = if File.directory?(opts[:require].to_s)
+          File.join(opts[:require], 'config')
+        else
+          File.join(options[:require], 'config')
+        end
+
+        %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          opts[:config_file] ||= path if File.exist?(path)
         end
       end
 
       # parse config file options
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
+      # set defaults
       opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]

lib/sidekiq/job_retry.rb

@@ -140,9 +140,7 @@ module Sidekiq
         queue
       end
 
-      # App code can stuff all sorts of crazy binary data into the error message
-      # that won't convert to JSON.
-      m = exception.message.to_s[0, 10_000]
+      m = exception_message(exception)
       if m.respond_to?(:scrub!)
         m.force_encoding("utf-8")
         m.scrub!
@@ -247,5 +245,17 @@ module Sidekiq
         exception_caused_by_shutdown?(e.cause, checked_causes)
     end
 
+    # Extract message from exception.
+    # Set a default if the message raises an error
+    def exception_message(exception)
+      begin
+        # App code can stuff all sorts of crazy binary data into the error message
+        # that won't convert to JSON.
+        exception.message.to_s[0, 10_000]
+      rescue
+        "!!! ERROR MESSAGE THREW AN ERROR !!!".dup
+      end
+    end
+
   end
 end

test/dummy/config/application.rb

@@ -7,9 +7,6 @@ module Dummy
     config.root = File.expand_path("../..", __FILE__)
     config.eager_load = false
     config.logger = Logger.new('/dev/null')
-
-    if Rails::VERSION::MAJOR >= 5
-      config.active_record.sqlite3.represent_boolean_as_integer = true
-    end
+    config.active_record.sqlite3.represent_boolean_as_integer = true
   end
 end

test/dummy/config/sidekiq.yml

@@ -1,2 +1,3 @@
 ---
+:require: ./test/fake_env.rb
 :concurrency: 25

test/helper.rb

@@ -3,11 +3,8 @@
 require 'bundler/setup'
 Bundler.require(:default, :test)
 
-require 'minitest/reporters'
 require 'minitest/autorun'
 
-MiniTest::Reporters.use! Minitest::Reporters::DefaultReporter.new
-
 $TESTING = true
 # disable minitest/parallel threads
 ENV["N"] = "0"

test/test_cli.rb

@@ -5,16 +5,18 @@ require 'sidekiq/cli'
 
 class TestCLI < Minitest::Test
   describe Sidekiq::CLI do
-    subject { Sidekiq::CLI.new }
 
-    let(:logdev) { StringIO.new }
+    def subject
+      @cli ||= Sidekiq::CLI.new
+    end
 
-    around do |test|
-      Sidekiq.stub :options, Sidekiq::DEFAULTS.dup do
-        Sidekiq.stub :logger, Sidekiq::Logger.new(logdev) do
-          test.call
-        end
-      end
+    def logdev
+      @logdev ||= StringIO.new
+    end
+
+    def setup
+      Sidekiq.options = Sidekiq::DEFAULTS.dup
+      Sidekiq.logger = Sidekiq::Logger.new(logdev)
     end
 
     describe '#parse' do
@@ -172,17 +174,37 @@ class TestCLI < Minitest::Test
             assert_equal 2, Sidekiq.options[:queues].count { |q| q == 'very_often' }
             assert_equal 1, Sidekiq.options[:queues].count { |q| q == 'seldom' }
           end
+        end
 
-          describe 'when config file is empty' do
-            it 'sets default options' do
-              subject.parse(%w[sidekiq -C ./test/config_empty.yml -r ./test/fake_env.rb])
+        describe 'default config file' do
+          describe 'when required path is a directory' do
+            it 'tries config/sidekiq.yml from required diretory' do
+              subject.parse(%w[sidekiq -r ./test/dummy])
 
-              assert_equal './test/config_empty.yml', Sidekiq.options[:config_file]
-              refute Sidekiq.options[:verbose]
-              assert_equal './test/fake_env.rb', Sidekiq.options[:require]
-              assert_nil Sidekiq.options[:environment]
-              assert_equal 10, Sidekiq.options[:concurrency]
-              assert_equal ['default'], Sidekiq.options[:queues]
+              assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file]
+              assert_equal 25, Sidekiq.options[:concurrency]
+            end
+          end
+
+          describe 'when required path is a file' do
+            it 'tries config/sidekiq.yml from current diretory' do
+              Sidekiq.options[:require] = './test/dummy' # stub current dir – ./
+
+              subject.parse(%w[sidekiq -r ./test/fake_env.rb])
+
+              assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file]
+              assert_equal 25, Sidekiq.options[:concurrency]
+            end
+          end
+
+          describe 'without any required path' do
+            it 'tries config/sidekiq.yml from current diretory' do
+              Sidekiq.options[:require] = './test/dummy' # stub current dir – ./
+
+              subject.parse(%w[sidekiq])
+
+              assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file]
+              assert_equal 25, Sidekiq.options[:concurrency]
             end
           end
 

test/test_retry.rb

@@ -10,6 +10,12 @@ class TestRetry < Minitest::Test
       include Sidekiq::Worker
     end
 
+    class BadErrorMessage < StandardError
+      def message
+        raise "Ahhh, this isn't supposed to happen"
+      end
+    end
+
     before do
       Sidekiq.redis {|c| c.flushdb }
     end
@@ -75,6 +81,19 @@ class TestRetry < Minitest::Test
       assert_equal "kerblammo! <20>", job["error_message"]
     end
 
+    # In the rare event that an error message raises an error itself,
+    # allow the job to retry. This will likely only happen for custom
+    # error classes that override #message
+    it 'handles error message that raises an error' do
+      assert_raises RuntimeError do
+        handler.local(worker, job, 'default') do
+          raise BadErrorMessage.new
+        end
+      end
+
+      assert_equal 1, Sidekiq::RetrySet.new.size
+      refute_nil job["error_message"]
+    end
 
     it 'allows a max_retries option in initializer' do
       max_retries = 7

test/test_sidekiq.rb

@@ -82,7 +82,7 @@ class TestSidekiq < Minitest::Test
     it 'does not continually retry' do
       assert_raises Redis::CommandError do
         Sidekiq.redis do |c|
-          raise Redis::CommandError, "READONLY You can't write against a read only slave."
+          raise Redis::CommandError, "READONLY You can't write against a replica."
         end
       end
     end
@@ -91,7 +91,7 @@ class TestSidekiq < Minitest::Test
       counts = []
       Sidekiq.redis do |c|
         counts << c.info['total_connections_received'].to_i
-        raise Redis::CommandError, "READONLY You can't write against a read only slave." if counts.size == 1
+        raise Redis::CommandError, "READONLY You can't write against a replica." if counts.size == 1
       end
       assert_equal 2, counts.size
       assert_equal counts[0] + 1, counts[1]