From d744250f764aa228b03c12cd94d088819458c59a Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 2 Jan 2019 09:23:34 -0800 Subject: [PATCH 01/21] fix require check --- lib/sidekiq/cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index afa23627..c8f17efb 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -238,7 +238,7 @@ module Sidekiq raise ArgumentError, "No such file #{opts[:config_file]}" end else - if File.directory?(opts[:require]) + if opts[:require] && File.directory?(opts[:require]) %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename| path = File.expand_path(filename, opts[:require]) opts[:config_file] ||= path if File.exist?(path) From d0887a0a0f949ca10a8f626327616392e386b806 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 2 Jan 2019 09:23:41 -0800 Subject: [PATCH 02/21] release --- Ent-Changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Ent-Changes.md b/Ent-Changes.md index cedf8d83..e45c6104 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -4,7 +4,7 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. -HEAD +1.7.2 ------------- - Add PT and JA translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language. From be28fc6e45a2cb3d054c89ad2bf598ab2ec5b64b Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 2 Jan 2019 09:37:37 -0800 Subject: [PATCH 03/21] switch to jruby so we auto-move to new versions --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a0bca02..b2a21d1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,4 +14,4 @@ rvm: - 2.3.7 - 2.4.4 - 2.5.1 - - jruby-9.1.17.0 + - jruby From 9c825f42747fd98eac7702980be62442669f2d34 Mon Sep 17 00:00:00 2001 From: Masato Ohba Date: Mon, 7 Jan 2019 02:27:06 +0900 Subject: [PATCH 04/21] Run tests against 2.6.0 (#4070) ref https://www.ruby-lang.org/en/news/2018/12/25/ruby-2-6-0-released/ --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b2a21d1d..baa38977 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,4 +14,5 @@ rvm: - 2.3.7 - 2.4.4 - 2.5.1 + - 2.6.0 - jruby From 432a800c45fe6d4fb4242b4dd8c776cdabdf6b9b Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Sun, 6 Jan 2019 09:28:14 -0800 Subject: [PATCH 05/21] remove ruby 2.2 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index baa38977..85a30711 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ gemfile: - gemfiles/rails_5.gemfile bundler_args: --without development load_test rvm: - - 2.2.10 - 2.3.7 - 2.4.4 - 2.5.1 From c1e8fc3282f6903ebb1e2d77a973adbf461f404f Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 7 Jan 2019 09:26:06 -0800 Subject: [PATCH 06/21] Redis bump warning, link to issue for more detail --- lib/sidekiq/cli.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index c8f17efb..9ea392a2 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -72,6 +72,7 @@ module Sidekiq # fire startup and start multithreading. ver = Sidekiq.redis_info['redis_version'] raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8' + logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4' # Since the user can pass us a connection pool explicitly in the initializer, we # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed. @@ -324,7 +325,7 @@ module Sidekiq o.on '-d', '--daemon', "Daemonize process" do |arg| opts[:daemon] = arg - puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services" + puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services" end o.on '-e', '--environment ENV', "Application environment" do |arg| @@ -364,12 +365,12 @@ module Sidekiq o.on '-L', '--logfile PATH', "path to writable logfile" do |arg| opts[:logfile] = arg - puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, Sidekiq will only log to STDOUT" + puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT" end o.on '-P', '--pidfile PATH', "path to pidfile" do |arg| opts[:pidfile] = arg - puts "WARNING: PID file creation will be removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services" + puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services" end o.on '-V', '--version', "Print version and exit" do |arg| From 1daa32dd9c6249f38764485cfdfdb4c1fd2e44f4 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 7 Jan 2019 09:31:22 -0800 Subject: [PATCH 07/21] bump, release --- Changes.md | 4 +++- lib/sidekiq/version.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Changes.md b/Changes.md index e2a84efc..62c2b421 100644 --- a/Changes.md +++ b/Changes.md @@ -2,9 +2,11 @@ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md) -HEAD +5.2.4 --------- +- Add warnings for various deprecations and changes coming in Sidekiq 6.0. + See the 6-0 branch. [#4056] - Various improvements to the Sidekiq test suite and coverage [#4026, #4039, Tensho] 5.2.3 diff --git a/lib/sidekiq/version.rb b/lib/sidekiq/version.rb index 283a1133..42c9d845 100644 --- a/lib/sidekiq/version.rb +++ b/lib/sidekiq/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true module Sidekiq - VERSION = "5.2.3" + VERSION = "5.2.4" end From d3fc2958d336f47c2dac991e3e9c3f7ae7a47d13 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Mon, 7 Jan 2019 16:30:27 -0800 Subject: [PATCH 08/21] modern terminology --- lib/sidekiq.rb | 4 ++-- test/test_sidekiq.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/sidekiq.rb b/lib/sidekiq.rb index 2aaeff68..39c2d73c 100644 --- a/lib/sidekiq.rb +++ b/lib/sidekiq.rb @@ -96,8 +96,8 @@ module Sidekiq begin yield conn rescue Redis::CommandError => ex - #2550 Failover can cause the server to become a slave, need - # to disconnect and reopen the socket to get back to the master. + #2550 Failover can cause the server to become a replica, need + # to disconnect and reopen the socket to get back to the primary. (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/ raise end diff --git a/test/test_sidekiq.rb b/test/test_sidekiq.rb index 391b0c0d..fe6887e2 100644 --- a/test/test_sidekiq.rb +++ b/test/test_sidekiq.rb @@ -82,7 +82,7 @@ class TestSidekiq < Minitest::Test it 'does not continually retry' do assert_raises Redis::CommandError do Sidekiq.redis do |c| - raise Redis::CommandError, "READONLY You can't write against a read only slave." + raise Redis::CommandError, "READONLY You can't write against a replica." end end end @@ -91,7 +91,7 @@ class TestSidekiq < Minitest::Test counts = [] Sidekiq.redis do |c| counts << c.info['total_connections_received'].to_i - raise Redis::CommandError, "READONLY You can't write against a read only slave." if counts.size == 1 + raise Redis::CommandError, "READONLY You can't write against a replica." if counts.size == 1 end assert_equal 2, counts.size assert_equal counts[0] + 1, counts[1] From a281aa44a63ea17e0b14f955ffd7d2c9ac147ef5 Mon Sep 17 00:00:00 2001 From: Andrew Babichev Date: Tue, 8 Jan 2019 18:20:59 +0100 Subject: [PATCH 09/21] Consider Default Required Path on Config File Search (#4077) * Consider default required path on config file search * Fix bundler at ci --- .travis.yml | 2 +- lib/sidekiq/cli.rb | 15 ++++++++++----- test/dummy/config/sidekiq.yml | 1 + test/test_cli.rb | 26 ++++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85a30711..ba22c227 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ cache: bundler services: - redis-server before_install: - - gem update --system + - gem install bundler -v '< 2' gemfile: - gemfiles/rails_4.gemfile - gemfiles/rails_5.gemfile diff --git a/lib/sidekiq/cli.rb b/lib/sidekiq/cli.rb index 9ea392a2..bb2f4c95 100644 --- a/lib/sidekiq/cli.rb +++ b/lib/sidekiq/cli.rb @@ -239,17 +239,22 @@ module Sidekiq raise ArgumentError, "No such file #{opts[:config_file]}" end else - if opts[:require] && File.directory?(opts[:require]) - %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename| - path = File.expand_path(filename, opts[:require]) - opts[:config_file] ||= path if File.exist?(path) - end + config_dir = if File.directory?(opts[:require].to_s) + File.join(opts[:require], 'config') + else + File.join(options[:require], 'config') + end + + %w[sidekiq.yml sidekiq.yml.erb].each do |config_file| + path = File.join(config_dir, config_file) + opts[:config_file] ||= path if File.exist?(path) end end # parse config file options opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file] + # set defaults opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty? opts[:strict] = true if opts[:strict].nil? opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"] diff --git a/test/dummy/config/sidekiq.yml b/test/dummy/config/sidekiq.yml index 774e4c88..b9e5211b 100644 --- a/test/dummy/config/sidekiq.yml +++ b/test/dummy/config/sidekiq.yml @@ -1,2 +1,3 @@ --- +:require: ./test/fake_env.rb :concurrency: 25 diff --git a/test/test_cli.rb b/test/test_cli.rb index 07aa9572..e8483644 100644 --- a/test/test_cli.rb +++ b/test/test_cli.rb @@ -247,10 +247,32 @@ class TestCLI < Minitest::Test describe 'default config file' do describe 'when required path is a directory' do - it 'tries config/sidekiq.yml' do + it 'tries config/sidekiq.yml from required diretory' do @cli.parse(%w[sidekiq -r ./test/dummy]) - assert_equal 'sidekiq.yml', File.basename(Sidekiq.options[:config_file]) + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] + assert_equal 25, Sidekiq.options[:concurrency] + end + end + + describe 'when required path is a file' do + it 'tries config/sidekiq.yml from current diretory' do + Sidekiq.options[:require] = './test/dummy' # stub current dir – ./ + + @cli.parse(%w[sidekiq -r ./test/fake_env.rb]) + + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] + assert_equal 25, Sidekiq.options[:concurrency] + end + end + + describe 'without any required path' do + it 'tries config/sidekiq.yml from current diretory' do + Sidekiq.options[:require] = './test/dummy' # stub current dir – ./ + + @cli.parse(%w[sidekiq]) + + assert_equal './test/dummy/config/sidekiq.yml', Sidekiq.options[:config_file] assert_equal 25, Sidekiq.options[:concurrency] end end From c2e819769a9d765b93c66f8ae0e2737885303ae1 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 8 Jan 2019 09:23:09 -0800 Subject: [PATCH 10/21] bump, changes --- Changes.md | 5 +++++ lib/sidekiq/version.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 62c2b421..488336b4 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,11 @@ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md) +5.2.5 +--------- + +- Fix default usage of `config/sidekiq.yml` [#4077, Tensho] + 5.2.4 --------- diff --git a/lib/sidekiq/version.rb b/lib/sidekiq/version.rb index 42c9d845..3928a10a 100644 --- a/lib/sidekiq/version.rb +++ b/lib/sidekiq/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true module Sidekiq - VERSION = "5.2.4" + VERSION = "5.2.5" end From 4769187c3da5e5d523d916b7374c7cd8d02d2387 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 9 Jan 2019 18:39:12 -0800 Subject: [PATCH 11/21] add default queue latency --- Ent-Changes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Ent-Changes.md b/Ent-Changes.md index e45c6104..d71c16f7 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -4,6 +4,11 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy. +HEAD +------------- + +- Add a gauge to the historical metrics for `default` queue latency [#4079] + 1.7.2 ------------- From 844f5a7f6a7d3dd8f27f2bf528fac3e642d70206 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Thu, 10 Jan 2019 12:19:48 -0800 Subject: [PATCH 12/21] changes --- Ent-Changes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Ent-Changes.md b/Ent-Changes.md index d71c16f7..ef021118 100644 --- a/Ent-Changes.md +++ b/Ent-Changes.md @@ -7,7 +7,9 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t HEAD ------------- -- Add a gauge to the historical metrics for `default` queue latency [#4079] +- Require Sidekiq Pro 4.0 and Sidekiq 5.2. +- Refactor historical metrics API to use revamped Statsd support in Sidekiq Pro +- Add a gauge to historical metrics for `default` queue latency [#4079] 1.7.2 ------------- From cdae00023f5093f3d3fd96dfb0684e2d6249766d Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 23 Jan 2019 20:04:12 -0800 Subject: [PATCH 13/21] Add clause for the Enterprise site license --- COMM-LICENSE | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/COMM-LICENSE b/COMM-LICENSE index c94e2722..3729b872 100644 --- a/COMM-LICENSE +++ b/COMM-LICENSE @@ -16,11 +16,13 @@ In order to use the Software under this Agreement, you must receive a “Source 1.3 Limited Organization License. If you purchased an Organization License (included with the Sidekiq Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. -1.4 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA. +1.4 Enterprise Site License. If you purchased a Site License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. -1.5 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. +1.5 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Contributed Systems of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Contributed Systems any royalties for your distribution of the Software in accordance with this EULA. -1.6 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). +1.6 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. + +1.7 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). 2. Modifications. Contributed Systems shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. From 3d8147cfaa9e8f234ee247c046ad70e189d6b189 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Wed, 23 Jan 2019 20:07:27 -0800 Subject: [PATCH 14/21] organization license -> limited license --- COMM-LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/COMM-LICENSE b/COMM-LICENSE index 3729b872..f9a6169f 100644 --- a/COMM-LICENSE +++ b/COMM-LICENSE @@ -14,7 +14,7 @@ In order to use the Software under this Agreement, you must receive a “Source 1.2 Unlimited Organization License. If you purchased an Organization License (included with the Sidekiq Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. -1.3 Limited Organization License. If you purchased an Organization License (included with the Sidekiq Enterprise Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. +1.3 Limited Enterprise License. If you purchased a Limited License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. The aggregate number of Workers run by the hosts must not exceed the maximum number of Workers authorized at the time of purchase. “Worker” means a thread within a Sidekiq server process which executes jobs. In order to run additional Workers, you must purchase an additional allowance from Contributed Systems. 1.4 Enterprise Site License. If you purchased a Site License for the Sidekiq Enterprise Software, you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. “Worker” means a thread within a Sidekiq server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers. From 23e6a5f756e620b12221d5369fb42d5d617c4b0f Mon Sep 17 00:00:00 2001 From: Brian Sewell Date: Thu, 31 Jan 2019 14:25:00 -0500 Subject: [PATCH 15/21] Rescue StandardErrors raised from exception's message (#4089) * Rescue standard errors raised from exception's message * Set a default message value if it raises an error * Update default message value * Add comments for exception_message --- lib/sidekiq/job_retry.rb | 16 +++++++++++++--- test/test_retry.rb | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq/job_retry.rb b/lib/sidekiq/job_retry.rb index b85bcb71..7851f65c 100644 --- a/lib/sidekiq/job_retry.rb +++ b/lib/sidekiq/job_retry.rb @@ -140,9 +140,7 @@ module Sidekiq queue end - # App code can stuff all sorts of crazy binary data into the error message - # that won't convert to JSON. - m = exception.message.to_s[0, 10_000] + m = exception_message(exception).dup if m.respond_to?(:scrub!) m.force_encoding("utf-8") m.scrub! @@ -247,5 +245,17 @@ module Sidekiq exception_caused_by_shutdown?(e.cause, checked_causes) end + # Extract message from exception. + # Set a default if the message raises an error + def exception_message(exception) + begin + # App code can stuff all sorts of crazy binary data into the error message + # that won't convert to JSON. + exception.message.to_s[0, 10_000] + rescue StandardError => e + "!!! ERROR MESSAGE THREW AN ERROR !!!" + end + end + end end diff --git a/test/test_retry.rb b/test/test_retry.rb index 15833fee..b78e9aed 100644 --- a/test/test_retry.rb +++ b/test/test_retry.rb @@ -10,6 +10,12 @@ class TestRetry < Minitest::Test include Sidekiq::Worker end + class BadErrorMessage < StandardError + def message + raise "Ahhh, this isn't supposed to happen" + end + end + before do Sidekiq.redis {|c| c.flushdb } end @@ -75,6 +81,19 @@ class TestRetry < Minitest::Test assert_equal "kerblammo! �", job["error_message"] end + # In the rare event that an error message raises an error itself, + # allow the job to retry. This will likely only happen for custom + # error classes that override #message + it 'handles error message that raises an error' do + assert_raises RuntimeError do + handler.local(worker, job, 'default') do + raise BadErrorMessage.new + end + end + + assert_equal 1, Sidekiq::RetrySet.new.size + refute_nil job["error_message"] + end it 'allows a max_retries option in initializer' do max_retries = 7 From 47028ef8b7cb998df6d7d72eb8af731bc6bbc341 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Thu, 31 Jan 2019 11:27:58 -0800 Subject: [PATCH 16/21] cleanup, #4089 --- lib/sidekiq/job_retry.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq/job_retry.rb b/lib/sidekiq/job_retry.rb index 7851f65c..c67548f4 100644 --- a/lib/sidekiq/job_retry.rb +++ b/lib/sidekiq/job_retry.rb @@ -140,7 +140,7 @@ module Sidekiq queue end - m = exception_message(exception).dup + m = exception_message(exception) if m.respond_to?(:scrub!) m.force_encoding("utf-8") m.scrub! @@ -252,8 +252,8 @@ module Sidekiq # App code can stuff all sorts of crazy binary data into the error message # that won't convert to JSON. exception.message.to_s[0, 10_000] - rescue StandardError => e - "!!! ERROR MESSAGE THREW AN ERROR !!!" + rescue + "!!! ERROR MESSAGE THREW AN ERROR !!!".dup end end From d392ebbb0e100dc5f58ef052f0fc9ad7473f64d8 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Feb 2019 10:23:18 -0800 Subject: [PATCH 17/21] super fetch latency --- Pro-Changes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pro-Changes.md b/Pro-Changes.md index 342cf8f2..158041d1 100644 --- a/Pro-Changes.md +++ b/Pro-Changes.md @@ -7,6 +7,8 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t HEAD --------- +- Increase super\_fetch retriever thread count from 1 to 2 to make it + less sensitive to Redis latency. - Better handling of invalid job JSON by reliable scheduler [#4053] - Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language. From f64e2d0753c33894648dd30374faad4c98994a1c Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Feb 2019 10:44:11 -0800 Subject: [PATCH 18/21] Simplify, prep for 6.0 --- .travis.yml | 6 ------ gemfiles/rails_4.gemfile | 31 ------------------------------- gemfiles/rails_5.gemfile | 31 ------------------------------- 3 files changed, 68 deletions(-) delete mode 100644 gemfiles/rails_4.gemfile delete mode 100644 gemfiles/rails_5.gemfile diff --git a/.travis.yml b/.travis.yml index ba22c227..01fb82e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,6 @@ sudo: false cache: bundler services: - redis-server -before_install: - - gem install bundler -v '< 2' -gemfile: - - gemfiles/rails_4.gemfile - - gemfiles/rails_5.gemfile -bundler_args: --without development load_test rvm: - 2.3.7 - 2.4.4 diff --git a/gemfiles/rails_4.gemfile b/gemfiles/rails_4.gemfile deleted file mode 100644 index 22678d11..00000000 --- a/gemfiles/rails_4.gemfile +++ /dev/null @@ -1,31 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -gem "rake" -gem "redis-namespace" -gem "rails", "~> 4.2" -gem "sqlite3", platforms: :ruby -gem "activerecord-jdbcsqlite3-adapter", "< 50", platforms: :jruby - -group :development do - gem "appraisal" -end - -group :test do - gem "minitest" - gem "minitest-focus" - gem "minitest-reporters" - gem "simplecov" -end - -group :development, :test do - gem "pry-byebug", platforms: :mri -end - -group :load_test do - gem "hiredis" - gem "toxiproxy" -end - -gemspec path: "../" diff --git a/gemfiles/rails_5.gemfile b/gemfiles/rails_5.gemfile deleted file mode 100644 index a3377873..00000000 --- a/gemfiles/rails_5.gemfile +++ /dev/null @@ -1,31 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -gem "rake" -gem "redis-namespace" -gem "rails", "~> 5.2" -gem "sqlite3", platforms: :ruby -gem "activerecord-jdbcsqlite3-adapter", ">= 50", platforms: :jruby - -group :development do - gem "appraisal" -end - -group :test do - gem "minitest" - gem "minitest-focus" - gem "minitest-reporters" - gem "simplecov" -end - -group :development, :test do - gem "pry-byebug", platforms: :mri -end - -group :load_test do - gem "hiredis" - gem "toxiproxy" -end - -gemspec path: "../" From 06d6e5c943d4dd9489fb972623c98bbf92253ad3 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Feb 2019 10:45:37 -0800 Subject: [PATCH 19/21] no longer necessary --- Appraisals | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 Appraisals diff --git a/Appraisals b/Appraisals deleted file mode 100644 index fe42abd4..00000000 --- a/Appraisals +++ /dev/null @@ -1,9 +0,0 @@ -appraise "rails-4" do - gem "rails", "~> 4.2" - gem 'activerecord-jdbcsqlite3-adapter', '< 50', platforms: :jruby -end - -appraise "rails-5" do - gem "rails", "~> 5.2" - gem 'activerecord-jdbcsqlite3-adapter', '>= 50', platforms: :jruby -end From d67e82473e80e04013077c18270ded165659cc67 Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Feb 2019 11:10:56 -0800 Subject: [PATCH 20/21] remove things which complicate the test suite --- Gemfile | 2 -- test/dummy/config/application.rb | 4 ---- test/helper.rb | 3 --- 3 files changed, 9 deletions(-) diff --git a/Gemfile b/Gemfile index c075c91e..fcd1d856 100644 --- a/Gemfile +++ b/Gemfile @@ -14,8 +14,6 @@ end group :test do gem 'minitest' - gem 'minitest-focus' - gem 'minitest-reporters' gem 'simplecov' end diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb index 5c73c4ca..c7a9d5d6 100644 --- a/test/dummy/config/application.rb +++ b/test/dummy/config/application.rb @@ -7,9 +7,5 @@ module Dummy config.root = File.expand_path("../..", __FILE__) config.eager_load = false config.logger = Logger.new('/dev/null') - - if Rails::VERSION::MAJOR >= 5 - config.active_record.sqlite3.represent_boolean_as_integer = true - end end end diff --git a/test/helper.rb b/test/helper.rb index da80a46a..f8fd467c 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -3,11 +3,8 @@ require 'bundler/setup' Bundler.require(:default, :test) -require 'minitest/reporters' require 'minitest/autorun' -MiniTest::Reporters.use! Minitest::Reporters::DefaultReporter.new - $TESTING = true # disable minitest/parallel threads ENV["N"] = "0" From f72d68faec46ade249dce60c060c68e7087f312c Mon Sep 17 00:00:00 2001 From: Mike Perham Date: Tue, 5 Feb 2019 11:44:49 -0800 Subject: [PATCH 21/21] adapt to rails 5.2.2's draconian sqlite3 version requirement --- Gemfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index fcd1d856..d54acb56 100644 --- a/Gemfile +++ b/Gemfile @@ -5,13 +5,9 @@ gemspec gem 'rake' gem 'redis-namespace' gem 'rails', '~> 5.2' -gem 'sqlite3', platforms: :ruby +gem 'sqlite3', '~> 1.3.6', platforms: :ruby gem 'activerecord-jdbcsqlite3-adapter', platforms: :jruby -group :development do - gem 'appraisal' -end - group :test do gem 'minitest' gem 'simplecov'