diff --git a/.travis.yml b/.travis.yml index f72679ef..46d0292e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,6 @@ sudo: false cache: bundler services: - redis-server -gemfile: - - gemfiles/rails_5.gemfile bundler_args: --without development load_test rvm: - 2.5.1 diff --git a/Appraisals b/Appraisals deleted file mode 100644 index 9fd54712..00000000 --- a/Appraisals +++ /dev/null @@ -1,4 +0,0 @@ -appraise "rails-5" do - gem "rails", "~> 5.2" - gem 'activerecord-jdbcsqlite3-adapter', '>= 50', platforms: :jruby -end diff --git a/COMM-LICENSE b/COMM-LICENSE index c94e2722..f9a6169f 100644 --- a/COMM-LICENSE +++ b/COMM-LICENSE @@ -14,13 +14,15 @@ In order to use the Software under this Agreement, you must receive a “Source 1.2 Unlimited Organization License. If you purchased an Organization License (included with the Sidekiq Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. -1.3 Limited Organization License. If you purchased an Organization License (included with the Sidekiq Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. +1.3 Limited Enterprise License. If you purchased a Limited License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. -1.4 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA. +1.4 Enterprise Site License. If you purchased a Site License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. -1.5 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. +1.5 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA. -1.6 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). +1.6 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. + +1.7 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). 2. Modifications. Contributed Systems shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. diff --git a/Changes.md b/Changes.md index 627f7eec..d6f6b769 100644 --- a/Changes.md +++ b/Changes.md @@ -2,7 +2,7 @@ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md) -HEAD +6.0 --------- This release has major breaking changes. Read and test carefully in production. @@ -23,9 +23,16 @@ get the old behavior. [#3968] arguments to Sidekiq. Use a proper process supervisor (e.g. systemd or foreman) to manage Sidekiq. +5.2.5 +--------- + +- Fix default usage of `config/sidekiq.yml` [#4077, Tensho] + 5.2.4 --------- +- Add warnings for various deprecations and changes coming in Sidekiq 6.0. + See the 6-0 branch. [#4056] - Various improvements to the Sidekiq test suite and coverage [#4026, #4039, Tensho] 5.2.3 diff --git a/Ent-Changes.md b/Ent-Changes.md index cedf8d83..ef021118 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -7,6 +7,13 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t HEAD ------------- +- Require Sidekiq Pro 4.0 and Sidekiq 5.2. +- Refactor historical metrics API to use revamped Statsd support in Sidekiq Pro +- Add a gauge to historical metrics for `default` queue latency [#4079] + +1.7.2 +------------- + - Add PT and JA translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language. - Fix elapsed time calculations to use monotonic clock [#4000, sj26] - Fix edge case where flapping leadership would cause old periodic diff --git a/Gemfile b/Gemfile index 7cf9174e..d54acb56 100644 --- a/Gemfile +++ b/Gemfile @@ -5,18 +5,11 @@ gemspec gem 'rake' gem 'redis-namespace' gem 'rails', '~> 5.2' -gem 'sqlite3', platforms: :ruby +gem 'sqlite3', '~> 1.3.6', platforms: :ruby gem 'activerecord-jdbcsqlite3-adapter', platforms: :jruby -group :development do - gem 'appraisal' -end - group :test do gem 'minitest' - gem 'minitest-around' - gem 'minitest-focus' - gem 'minitest-reporters' gem 'simplecov' end diff --git a/Pro-Changes.md b/Pro-Changes.md index 342cf8f2..158041d1 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -7,6 +7,8 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t HEAD --------- +- Increase super\_fetch retriever thread count from 1 to 2 to make it + less sensitive to Redis latency. - Better handling of invalid job JSON by reliable scheduler [#4053] - Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language. diff --git a/gemfiles/rails_5.gemfile b/gemfiles/rails_5.gemfile deleted file mode 100644 index ea2840aa..00000000 --- a/gemfiles/rails_5.gemfile +++ /dev/null @@ -1,32 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -gem "rake" -gem "redis-namespace" -gem "rails", "~> 5.2" -gem "sqlite3", platforms: :ruby -gem "activerecord-jdbcsqlite3-adapter", ">= 50", platforms: :jruby - -group :development do - gem "appraisal" -end - -group :test do - gem "minitest" - gem "minitest-around" - gem "minitest-focus" - gem "minitest-reporters" - gem "simplecov" -end - -group :development, :test do - gem "pry-byebug", platforms: :mri -end - -group :load_test do - gem "hiredis" - gem "toxiproxy" -end - -gemspec path: "../" diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index 97944827..b59937aa 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -62,6 +62,7 @@ module Sidekiq # fire startup and start multithreading. ver = Sidekiq.redis_info['redis_version'] raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8' + logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4' # Since the user can pass us a connection pool explicitly in the initializer, we # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed. @@ -197,17 +198,22 @@ module Sidekiq raise ArgumentError, "No such file #{opts[:config_file]}" end else - if opts[:require] && File.directory?(opts[:require]) - %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename| - path = File.expand_path(filename, opts[:require]) - opts[:config_file] ||= path if File.exist?(path) - end + config_dir = if File.directory?(opts[:require].to_s) + File.join(opts[:require], 'config') + else + File.join(options[:require], 'config') + end + + %w[sidekiq.yml sidekiq.yml.erb].each do |config_file| + path = File.join(config_dir, config_file) + opts[:config_file] ||= path if File.exist?(path) end end # parse config file options opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file] + # set defaults opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty? opts[:strict] = true if opts[:strict].nil? opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"] diff --git a/lib/sidekiq/job_retry.rb b/lib/sidekiq/job_retry.rb index b85bcb71..c67548f4 100644 --- a/lib/sidekiq/job_retry.rb +++ b/lib/sidekiq/job_retry.rb @@ -140,9 +140,7 @@ module Sidekiq queue end - # App code can stuff all sorts of crazy binary data into the error message - # that won't convert to JSON. - m = exception.message.to_s[0, 10_000] + m = exception_message(exception) if m.respond_to?(:scrub!) m.force_encoding("utf-8") m.scrub! @@ -247,5 +245,17 @@ module Sidekiq exception_caused_by_shutdown?(e.cause, checked_causes) end + # Extract message from exception. + # Set a default if the message raises an error + def exception_message(exception) + begin + # App code can stuff all sorts of crazy binary data into the error message + # that won't convert to JSON. + exception.message.to_s[0, 10_000] + rescue + "!!! ERROR MESSAGE THREW AN ERROR !!!".dup + end + end + end end diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb index 5c73c4ca..a0e634cb 100644 --- a/test/dummy/config/application.rb +++ b/test/dummy/config/application.rb @@ -7,9 +7,6 @@ module Dummy config.root = File.expand_path("../..", __FILE__) config.eager_load = false config.logger = Logger.new('/dev/null') - - if Rails::VERSION::MAJOR >= 5 - config.active_record.sqlite3.represent_boolean_as_integer = true - end + config.active_record.sqlite3.represent_boolean_as_integer = true end end diff --git a/test/dummy/config/sidekiq.yml b/test/dummy/config/sidekiq.yml index 774e4c88..b9e5211b 100644 --- a/test/dummy/config/sidekiq.yml +++ b/test/dummy/config/sidekiq.yml @@ -1,2 +1,3 @@ --- +:require: ./test/fake_env.rb :concurrency: 25 diff --git a/test/helper.rb b/test/helper.rb index da80a46a..f8fd467c 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -3,11 +3,8 @@ require 'bundler/setup' Bundler.require(:default, :test) -require 'minitest/reporters' require 'minitest/autorun' -MiniTest::Reporters.use! Minitest::Reporters::DefaultReporter.new - $TESTING = true # disable minitest/parallel threads ENV["N"] = "0" diff --git a/test/test_cli.rb b/test/test_cli.rb index 52d9e6e7..c54c7b2b 100644 --- a/test/test_cli.rb +++ b/test/test_cli.rb @@ -5,16 +5,18 @@ require 'sidekiq/cli' class TestCLI < Minitest::Test describe Sidekiq::CLI do - subject { Sidekiq::CLI.new } - let(:logdev) { StringIO.new } + def subject + @cli ||= Sidekiq::CLI.new + end - around do |test| - Sidekiq.stub :options, Sidekiq::DEFAULTS.dup do - Sidekiq.stub :logger, Sidekiq::Logger.new(logdev) do - test.call - end - end + def logdev + @logdev ||= StringIO.new + end + + def setup + Sidekiq.options = Sidekiq::DEFAULTS.dup + Sidekiq.logger = Sidekiq::Logger.new(logdev) end describe '#parse' do @@ -172,17 +174,37 @@ class TestCLI < Minitest::Test assert_equal 2, Sidekiq.options[:queues].count { |q| q == 'very_often' } assert_equal 1, Sidekiq.options[:queues].count { |q| q == 'seldom' } end + end - describe 'when config file is empty' do - it 'sets default options' do - subject.parse(%w[sidekiq -C ./test/config_empty.yml -r ./test/fake_env.rb]) + describe 'default config file' do + describe 'when required path is a directory' do + it 'tries config/sidekiq.yml from required diretory' do + subject.parse(%w[sidekiq -r ./test/dummy]) - assert_equal './test/config_empty.yml', Sidekiq.options[:config_file] - refute Sidekiq.options[:verbose] - assert_equal './test/fake_env.rb', Sidekiq.options[:require] - assert_nil Sidekiq.options[:environment] - assert_equal 10, Sidekiq.options[:concurrency] - assert_equal ['default'], Sidekiq.options[:queues] + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] + assert_equal 25, Sidekiq.options[:concurrency] + end + end + + describe 'when required path is a file' do + it 'tries config/sidekiq.yml from current diretory' do + Sidekiq.options[:require] = './test/dummy' # stub current dir – ./ + + subject.parse(%w[sidekiq -r ./test/fake_env.rb]) + + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] + assert_equal 25, Sidekiq.options[:concurrency] + end + end + + describe 'without any required path' do + it 'tries config/sidekiq.yml from current diretory' do + Sidekiq.options[:require] = './test/dummy' # stub current dir – ./ + + subject.parse(%w[sidekiq]) + + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] + assert_equal 25, Sidekiq.options[:concurrency] end end diff --git a/test/test_retry.rb b/test/test_retry.rb index 21321fda..a7da3837 100644 --- a/test/test_retry.rb +++ b/test/test_retry.rb @@ -10,6 +10,12 @@ class TestRetry < Minitest::Test include Sidekiq::Worker end + class BadErrorMessage < StandardError + def message + raise "Ahhh, this isn't supposed to happen" + end + end + before do Sidekiq.redis {|c| c.flushdb } end @@ -75,6 +81,19 @@ class TestRetry < Minitest::Test assert_equal "kerblammo! �", job["error_message"] end + # In the rare event that an error message raises an error itself, + # allow the job to retry. This will likely only happen for custom + # error classes that override #message + it 'handles error message that raises an error' do + assert_raises RuntimeError do + handler.local(worker, job, 'default') do + raise BadErrorMessage.new + end + end + + assert_equal 1, Sidekiq::RetrySet.new.size + refute_nil job["error_message"] + end it 'allows a max_retries option in initializer' do max_retries = 7 diff --git a/test/test_sidekiq.rb b/test/test_sidekiq.rb index 391b0c0d..fe6887e2 100644 --- a/test/test_sidekiq.rb +++ b/test/test_sidekiq.rb @@ -82,7 +82,7 @@ class TestSidekiq < Minitest::Test it 'does not continually retry' do assert_raises Redis::CommandError do Sidekiq.redis do |c| - raise Redis::CommandError, "READONLY You can't write against a read only slave." + raise Redis::CommandError, "READONLY You can't write against a replica." end end end @@ -91,7 +91,7 @@ class TestSidekiq < Minitest::Test counts = [] Sidekiq.redis do |c| counts << c.info['total_connections_received'].to_i - raise Redis::CommandError, "READONLY You can't write against a read only slave." if counts.size == 1 + raise Redis::CommandError, "READONLY You can't write against a replica." if counts.size == 1 end assert_equal 2, counts.size assert_equal counts[0] + 1, counts[1]